fix pattern
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index 5faa478..e4790ff 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -2384,13 +2384,15 @@ } // End HasPackedD16VMem,HasMTBUFInsts. multiclass MTBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt, - string opcode, ValueType memoryVt = vt> { + string opcode, ValueType memoryVt = vt, bit isTrue16 = false> { defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>); def : GCNPat< (st vt:$vdata, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, timm:$format, timm:$auxiliary, 0), - (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, + (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) + !if(isTrue16, (lo16_to_vgpr32 $vdata), (vt getVregSrcForVT<vt>.ret:$vdata)), + SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (as_i8timm $format), (extract_cpol $auxiliary), (extract_swz $auxiliary)) >; @@ -2398,7 +2400,9 @@ def : GCNPat< (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, timm:$format, timm:$auxiliary, timm), - (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, + (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) + !if(isTrue16, (lo16_to_vgpr32 $vdata), (vt getVregSrcForVT<vt>.ret:$vdata)), + VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (as_i8timm $format), (extract_cpol $auxiliary), (extract_swz $auxiliary)) >; @@ -2406,7 +2410,9 @@ def : GCNPat< (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, timm:$format, timm:$auxiliary, 0), - (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, + (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) + !if(isTrue16, (lo16_to_vgpr32 $vdata), (vt getVregSrcForVT<vt>.ret:$vdata)), + VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (as_i8timm $format), (extract_cpol $auxiliary), (extract_swz $auxiliary)) >; @@ -2415,7 +2421,7 @@ (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, timm:$format, timm:$auxiliary, timm), (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) - getVregSrcForVT<vt>.ret:$vdata, + !if(isTrue16, (lo16_to_vgpr32 $vdata), (vt getVregSrcForVT<vt>.ret:$vdata)), (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (as_i8timm $format), (extract_cpol $auxiliary), (extract_swz $auxiliary)) @@ -2423,11 +2429,11 @@ } multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, - string opcode, ValueType memoryVt = vt> { + string opcode, ValueType memoryVt = vt, bit isTrue16 = false> { let SubtargetPredicate = HasUnrestrictedSOffset in { - defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode, memoryVt>; + defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode, memoryVt, isTrue16>; } - defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; + defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt, isTrue16>; } let OtherPredicates = [HasMTBUFInsts] in { @@ -2457,6 +2463,9 @@ let OtherPredicates = [HasPackedD16VMem,HasMTBUFInsts] in { foreach vt = Reg16Types.types in { + let True16Predicate = UseRealTrue16Insts in + defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, vt, "TBUFFER_STORE_FORMAT_D16_X", vt, /*isTrue16*/true>; + let True16Predicate = NotUseRealTrue16Insts in defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, vt, "TBUFFER_STORE_FORMAT_D16_X">; } defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">;
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index eb23b6f..68d258e 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3161,16 +3161,16 @@ let True16Predicate = UseRealTrue16Insts in { def : GCNPat < (i16 (DivergentSextInreg<i1> i16:$src)), - (V_BFE_I32_e64 + (EXTRACT_SUBREG (V_BFE_I32_e64 (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (i16 (IMPLICIT_DEF)), hi16), - (i32 0), (i32 1)) + (i32 0), (i32 1)), lo16) >; def : GCNPat < (i16 (DivergentSextInreg<i8> i16:$src)), - (V_BFE_I32_e64 + (EXTRACT_SUBREG (V_BFE_I32_e64 (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (i16 (IMPLICIT_DEF)), hi16), - (i32 0), (i32 8)) + (i32 0), (i32 8)) , lo16) >; } @@ -3742,12 +3742,15 @@ let True16Predicate = UseRealTrue16Insts in { def : GCNPat < (i16 (bswap i16:$a)), - (EXTRACT_SUBREG (V_PERM_B32_e64 (i32 0), (COPY VGPR_16:$a), (S_MOV_B32 (i32 0x0c0c0001))), lo16) + (EXTRACT_SUBREG (V_PERM_B32_e64 (i32 0), + (REG_SEQUENCE VGPR_32, VGPR_16:$a, lo16, (i16 (IMPLICIT_DEF)), hi16), + (S_MOV_B32 (i32 0x0c0c0001))), lo16) >; def : GCNPat < (i32 (zext (bswap i16:$a))), - (V_PERM_B32_e64 (i32 0), (COPY VGPR_16:$a), (S_MOV_B32 (i32 0x0c0c0001))) + (V_PERM_B32_e64 (i32 0), (REG_SEQUENCE VGPR_32, $a, lo16, (IMPLICIT_DEF), hi16), + (S_MOV_B32 (i32 0x0c0c0001))) >; } @@ -4000,7 +4003,8 @@ let True16Predicate = UseRealTrue16Insts in { def : GCNPat < (v2i16 (DivergentBinFrag<build_vector> (i16 0), (i16 VGPR_16:$src1))), - (v2i16 (V_LSHLREV_B32_e64 (i16 16), VGPR_16:$src1)) + (v2i16 (V_LSHLREV_B32_e64 (i16 16), + (REG_SEQUENCE VGPR_32, $src1, lo16, (i16 (IMPLICIT_DEF)), hi16))) >; def : GCNPat <
diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td index bcf2ca6..f132d2e 100644 --- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td +++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
@@ -1393,6 +1393,13 @@ $src) >; +class ZExt_i16_i1_Pat_t16 <SDNode ext> : GCNPat < + (i16 (ext i1:$src)), + (V_CNDMASK_B16_t16_e64 0/*src0mod*/, 0/*src0*/, + 0/*src1mod*/, 1/*src1*/, + $src) +>; + let True16Predicate = NotUseRealTrue16Insts in { def : GCNPat < (and i16:$src0, i16:$src1), @@ -1453,17 +1460,32 @@ } // End Predicates = [Has16BitInsts, isGFX8GFX9] -let Predicates = [Has16BitInsts] in { +let OtherPredicates = [Has16BitInsts] in { +let True16Predicate = NotUseRealTrue16Insts in { def : ZExt_i16_i1_Pat<zext>; def : ZExt_i16_i1_Pat<anyext>; +} +let True16Predicate = UseRealTrue16Insts in { +def : ZExt_i16_i1_Pat_t16<zext>; +def : ZExt_i16_i1_Pat_t16<anyext>; +} + +let True16Predicate = NotUseRealTrue16Insts in def : GCNPat < (i16 (sext i1:$src)), (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), /*src1mod*/(i32 0), /*src1*/(i32 -1), $src) >; +let True16Predicate = UseRealTrue16Insts in +def : GCNPat < + (i16 (sext i1:$src)), + (V_CNDMASK_B16_t16_e64 /*src0mod*/0, /*src0*/0, + /*src1mod*/0, /*src1*/-1, $src) +>; + } // End Predicates = [Has16BitInsts]
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td index 05f47f6..73c5f5c 100644 --- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td +++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -1807,6 +1807,10 @@ def : GCNPat<(v2bf16 (build_vector (bf16 (bf16_fpround (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))), (bf16 (bf16_fpround (f32 (VOP3Mods f32:$src1, i32:$src1_modifiers)))))), (V_CVT_PK_BF16_F32_e64 $src0_modifiers, $src0, $src1_modifiers, $src1)>; + let True16Predicate = UseRealTrue16Insts in + def : GCNPat<(bf16 (bf16_fpround (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))), + (EXTRACT_SUBREG (V_CVT_PK_BF16_F32_e64 $src0_modifiers, $src0, 0, (f32 (IMPLICIT_DEF))), lo16)>; + let True16Predicate = NotUseRealTrue16Insts in def : GCNPat<(bf16 (bf16_fpround (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))), (V_CVT_PK_BF16_F32_e64 $src0_modifiers, $src0, 0, (f32 (IMPLICIT_DEF)))>; }
diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td index a379785..f52d014 100644 --- a/llvm/lib/Target/AMDGPU/VOPInstructions.td +++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -2406,8 +2406,8 @@ >; class ClassPat_t16<Instruction inst, ValueType vt> : GCNPat < - (i1 (is_fpclass (vt (VOP3ModsNonCanonicalizing vt:$src0, i32:$src0_mods)), (i32 timm:$mask))), - (inst i32:$src0_mods, vt:$src0, SRCMODS.NONE, (V_MOV_B32_e32 timm:$mask)) + (i1 (is_fpclass (vt (VOP3ModsNonCanonicalizing vt:$src0, i32:$src0_mods)), (i16 timm:$mask))), + (inst i32:$src0_mods, vt:$src0, SRCMODS.NONE, (V_MOV_B16_t16_e64 0, timm:$mask, 0)) >; def : ClassPat<V_CMP_CLASS_F16_e64, f16> {
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll index 991071a..46f5b6f 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -2936,85 +2936,79 @@ ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v8 :: v_dual_add_f32 v6, 0x40c00000, v6 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v9.l -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v6, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v8 :: v_dual_add_f32 v5, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4 +; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v8, v12 :: v_dual_add_f32 v6, 0x40c00000, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v6, 16, 1 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v2, v7, v8 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7 +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v7, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v7, 0x7fff ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v8, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v11, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v7 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v6.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l ; GFX11-TRUE16-NEXT: .LBB22_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -7773,85 +7767,79 @@ ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v8 :: v_dual_add_f32 v6, 0x40c00000, v6 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v9.l -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v6, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v8 :: v_dual_add_f32 v5, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4 +; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v8, v12 :: v_dual_add_f32 v6, 0x40c00000, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v6, 16, 1 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v2, v7, v8 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7 +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v7, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v7, 0x7fff ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v8, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v11, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v7 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v6.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l ; GFX11-TRUE16-NEXT: .LBB46_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -12206,85 +12194,79 @@ ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v8 :: v_dual_add_f32 v6, 0x40c00000, v6 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v9.l -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v6, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v8 :: v_dual_add_f32 v5, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4 +; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v8, v12 :: v_dual_add_f32 v6, 0x40c00000, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v6, 16, 1 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v2, v7, v8 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7 +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v7, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v7, 0x7fff ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v8, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v11, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v7 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v6.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l ; GFX11-TRUE16-NEXT: .LBB66_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -16195,85 +16177,79 @@ ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v8 :: v_dual_add_f32 v6, 0x40c00000, v6 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v9.l -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v6, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v8 :: v_dual_add_f32 v5, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4 +; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v8, v12 :: v_dual_add_f32 v6, 0x40c00000, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v6, 16, 1 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v2, v7, v8 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7 +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v7, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v7, 0x7fff ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v8, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v11, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v7 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v6.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l ; GFX11-TRUE16-NEXT: .LBB82_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -19804,89 +19780,82 @@ ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB94_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v4, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v12, v5, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v11, vcc_lo +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v10, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v7, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v1, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v11, v12 :: v_dual_add_f32 v0, 0x40c00000, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v3, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v13, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v6, 0x40c00000, v6 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v7 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6 -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v6, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v5, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v8, v13 :: v_dual_and_b32 v2, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v8, 16, 1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v8, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v9, v13, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v14, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v14, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v8 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v9.l ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v12, v15, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v7.l +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v12, v15, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v8.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v9.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l ; GFX11-TRUE16-NEXT: .LBB94_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -22926,87 +22895,79 @@ ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v4, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v4, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4 -; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v4, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v11, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v8.l -; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v4, v7 :: v_dual_add_f32 v6, 0x40c00000, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v6, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v9, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v8 :: v_dual_and_b32 v5, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v11, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v10, v12 :: v_dual_and_b32 v6, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v6 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v10 ; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1 ; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v11, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v11, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6 +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v11, v7, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v9, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v13, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v13, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v6.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v7.l ; GFX11-TRUE16-NEXT: .LBB102_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 @@ -25852,94 +25813,87 @@ ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4 ; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v19 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v18 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v19.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v17.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v18.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v17 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v18 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1 ; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v19 -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_cndmask_b32 v0, v5, v6 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v9, v7 :: v_dual_and_b32 v9, 0xffff0000, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v10, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v1 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v8, v10 :: v_dual_lshlrev_b32 v1, 16, v17 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add3_u32 v0, v7, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v16 +; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v8 -; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v9, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9 -; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v8, 16, 1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v6.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v4 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v0, v7 :: v_dual_add_f32 v9, 0x40c00000, v10 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v5, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v9, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v2.l -; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v8, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v9, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v9, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v8 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v10, v8, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v14.l -; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19] -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v3 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v10.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16 ; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll index ffff9da..6095375 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
@@ -144,39 +144,38 @@ ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB0_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 +; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0x7fc0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l ; GFX11-TRUE16-NEXT: .LBB0_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -906,38 +905,39 @@ ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB4_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_add_f32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v3, v7 :: v_dual_mov_b32 v3, 0x7fc0 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, 0x7fc0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l ; GFX11-TRUE16-NEXT: .LBB4_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -1330,7 +1330,7 @@ ; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GFX9-NEXT: ; %bb.1: ; %cmp.true -; GFX9-NEXT: v_pk_add_u16 v1, v1, 3 +; GFX9-NEXT: v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0] ; GFX9-NEXT: v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0] ; GFX9-NEXT: ; %bb.2: ; %end ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] @@ -1345,7 +1345,7 @@ ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-NEXT: ; %bb.1: ; %cmp.true -; GFX11-NEXT: v_pk_add_u16 v1, v1, 3 +; GFX11-NEXT: v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0] ; GFX11-NEXT: v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0] ; GFX11-NEXT: ; %bb.2: ; %end ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 @@ -1452,7 +1452,7 @@ ; GFX9-NEXT: s_cmp_lg_u32 s4, 1 ; GFX9-NEXT: s_cbranch_scc1 .LBB7_5 ; GFX9-NEXT: ; %bb.4: ; %cmp.true -; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 +; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0] ; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; GFX9-NEXT: .LBB7_5: @@ -1475,7 +1475,7 @@ ; GFX11-NEXT: s_cmp_lg_u32 s2, 1 ; GFX11-NEXT: s_cbranch_scc1 .LBB7_4 ; GFX11-NEXT: ; %bb.3: ; %cmp.true -; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 +; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0] ; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; GFX11-NEXT: .LBB7_4: @@ -1789,7 +1789,7 @@ ; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] ; GFX9-NEXT: ; %bb.1: ; %cmp.true -; GFX9-NEXT: v_pk_add_u16 v1, v1, 3 +; GFX9-NEXT: v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0] ; GFX9-NEXT: v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0] ; GFX9-NEXT: ; %bb.2: ; %end ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5] @@ -1804,7 +1804,7 @@ ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-NEXT: ; %bb.1: ; %cmp.true -; GFX11-NEXT: v_pk_add_u16 v1, v1, 3 +; GFX11-NEXT: v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0] ; GFX11-NEXT: v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0] ; GFX11-NEXT: ; %bb.2: ; %end ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 @@ -1912,7 +1912,7 @@ ; GFX9-NEXT: s_cmp_lg_u32 s4, 1 ; GFX9-NEXT: s_cbranch_scc1 .LBB11_5 ; GFX9-NEXT: ; %bb.4: ; %cmp.true -; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 +; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0] ; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0] ; GFX9-NEXT: s_setpc_b64 s[30:31] ; GFX9-NEXT: .LBB11_5: @@ -1935,7 +1935,7 @@ ; GFX11-NEXT: s_cmp_lg_u32 s2, 1 ; GFX11-NEXT: s_cbranch_scc1 .LBB11_4 ; GFX11-NEXT: ; %bb.3: ; %cmp.true -; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 +; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0] ; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; GFX11-NEXT: .LBB11_4:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll index 8b1ab66..c7d106b 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -2401,42 +2401,41 @@ ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0 ; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1 ; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v10, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v1, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v0, 0x7fff ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 @@ -5878,42 +5877,41 @@ ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0 ; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1 ; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v10, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v1, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v0, 0x7fff ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 @@ -9010,42 +9008,41 @@ ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0 ; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1 ; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v10, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v1, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v0, 0x7fff ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 @@ -11782,42 +11779,41 @@ ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0 ; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1 ; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v10, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v1, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v0, 0x7fff ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 @@ -14182,50 +14178,44 @@ ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB94_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v6, v7 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v2, 16, v0 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v11, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v6, v9 :: v_dual_add_f32 v0, 0x40c00000, v0 +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l ; GFX11-TRUE16-NEXT: .LBB94_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -16168,42 +16158,41 @@ ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0 ; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 ; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v8, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v4, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l ; GFX11-TRUE16-NEXT: .LBB102_2: ; %end @@ -17975,50 +17964,47 @@ ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4 ; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v9 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v9 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v8 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v2, v3, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v6, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v7 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v1, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v10, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v0 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v2.l -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll index c421dcf..8e053d8 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -2179,63 +2179,62 @@ ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB10_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2 ; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc_lo -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v6, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v8, v6 :: v_dual_add_f32 v5, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4 ; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v8, vcc_lo ; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_cndmask_b32 v5, v7, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v10, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v11, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v5.l ; GFX11-TRUE16-NEXT: .LBB10_2: ; %end @@ -5590,63 +5589,62 @@ ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2 ; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc_lo -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v6, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v8, v6 :: v_dual_add_f32 v5, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4 ; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v8, vcc_lo ; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_cndmask_b32 v5, v7, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v10, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v11, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v5.l ; GFX11-TRUE16-NEXT: .LBB26_2: ; %end @@ -8297,65 +8295,64 @@ ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB38_4 ; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v14 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v14 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v13 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v15.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, 0x7fc07fc0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v7, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v9, v10 :: v_dual_lshlrev_b32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, 0x7fc07fc0 :: v_dual_lshlrev_b32 v3, 16, v13 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v15 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v4 :: v_dual_add_f32 v4, 0x40c00000, v5 -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v6.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v14 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v10, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v1, v7, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v15 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v5, 16, v15 +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_cndmask_b32 v4, v7, v8 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14 +; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v1 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v4 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v2.l ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) @@ -12222,60 +12219,59 @@ ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v0, 0x40c00000, v0 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0 ; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v6, v7 :: v_dual_add_f32 v4, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v0, v7, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v6, v7 :: v_dual_add_f32 v5, 0x40c00000, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v6, vcc_lo ; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v5, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_cndmask_b32 v5, v8, v9 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v10, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l ; GFX11-TRUE16-NEXT: .LBB48_2: ; %end @@ -13330,53 +13326,51 @@ ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB52_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v4, 16, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v6, v7 :: v_dual_and_b32 v1, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v7, v11 :: v_dual_and_b32 v1, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v12, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll index 04f1b40..1971fef 100644 --- a/llvm/test/CodeGen/AMDGPU/bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -29132,29 +29132,16 @@ ; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; -; GFX1250TRUE16-LABEL: v_frexp_bf16_i16: -; GFX1250TRUE16: ; %bb.0: -; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250TRUE16-NEXT: v_frexp_mant_f32_e32 v1, v0 -; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v1, s0 -; GFX1250TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v1, v0 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l -; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31] -; -; GFX1250FAKE16-LABEL: v_frexp_bf16_i16: -; GFX1250FAKE16: ; %bb.0: -; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1250FAKE16-NEXT: v_frexp_mant_f32_e32 v0, v1 -; GFX1250FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v1, v1 -; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 -; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31] +; GFX1250-LABEL: v_frexp_bf16_i16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_frexp_mant_f32_e32 v0, v1 +; GFX1250-NEXT: v_frexp_exp_i32_f32_e32 v1, v1 +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %op = call { bfloat, i16 } @llvm.frexp.bf16.i16(bfloat %a) ret { bfloat, i16 } %op } @@ -29493,28 +29480,51 @@ ; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_log2_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 -; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo -; GFX11-NEXT: v_ldexp_f32 v0, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_log_f32_e32 v0, v0 -; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0) -; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1 -; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1 -; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0 -; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff -; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_log2_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo +; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_log_f32_e32 v0, v0 +; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0) +; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_log2_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo +; GFX11FAKE16-NEXT: v_ldexp_f32 v0, v0, v2 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_log_f32_e32 v0, v0 +; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0) +; GFX11FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX1250TRUE16-LABEL: v_log2_bf16: ; GFX1250TRUE16: ; %bb.0: @@ -29924,37 +29934,69 @@ ; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_exp_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0 -; GFX11-NEXT: v_rndne_f32_e32 v2, v1 -; GFX11-NEXT: v_fma_f32 v3, 0x3fb8aa3b, v0, -v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_sub_f32_e32 v1, v1, v2 -; GFX11-NEXT: v_fmamk_f32 v3, v0, 0x32a5705f, v3 -; GFX11-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GFX11-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_add_f32_e32 v1, v1, v3 -; GFX11-NEXT: v_exp_f32_e32 v1, v1 -; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0) -; GFX11-NEXT: v_ldexp_f32 v1, v1, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo -; GFX11-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0 -; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1 -; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0 -; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_exp_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0 +; GFX11TRUE16-NEXT: v_fma_f32 v2, 0x3fb8aa3b, v0, -v1 +; GFX11TRUE16-NEXT: v_rndne_f32_e32 v3, v1 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v3 +; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0 +; GFX11TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x32a5705f, v2 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3 +; GFX11TRUE16-NEXT: v_exp_f32_e32 v1, v1 +; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0) +; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v2 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo +; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0 +; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_exp_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0 +; GFX11FAKE16-NEXT: v_rndne_f32_e32 v2, v1 +; GFX11FAKE16-NEXT: v_fma_f32 v3, 0x3fb8aa3b, v0, -v1 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11FAKE16-NEXT: v_sub_f32_e32 v1, v1, v2 +; GFX11FAKE16-NEXT: v_fmamk_f32 v3, v0, 0x32a5705f, v3 +; GFX11FAKE16-NEXT: v_cvt_i32_f32_e32 v2, v2 +; GFX11FAKE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_add_f32_e32 v1, v1, v3 +; GFX11FAKE16-NEXT: v_exp_f32_e32 v1, v1 +; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0) +; GFX11FAKE16-NEXT: v_ldexp_f32 v1, v1, v2 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo +; GFX11FAKE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0 +; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX1250TRUE16-LABEL: v_exp_bf16: ; GFX1250TRUE16: ; %bb.0: @@ -30129,28 +30171,51 @@ ; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_exp2_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0 -; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo -; GFX11-NEXT: v_add_f32_e32 v0, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_exp_f32_e32 v0, v0 -; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0) -; GFX11-NEXT: v_ldexp_f32 v0, v0, v1 -; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1 -; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0 -; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff -; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_exp2_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo +; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0) +; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_exp2_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0 +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo +; GFX11FAKE16-NEXT: v_add_f32_e32 v0, v0, v2 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0) +; GFX11FAKE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX1250TRUE16-LABEL: v_exp2_bf16: ; GFX1250TRUE16: ; %bb.0: @@ -30334,37 +30399,69 @@ ; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_exp10_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0 -; GFX11-NEXT: v_rndne_f32_e32 v2, v1 -; GFX11-NEXT: v_fma_f32 v3, 0x40549a78, v0, -v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_sub_f32_e32 v1, v1, v2 -; GFX11-NEXT: v_fmamk_f32 v3, v0, 0x33979a37, v3 -; GFX11-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GFX11-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_add_f32_e32 v1, v1, v3 -; GFX11-NEXT: v_exp_f32_e32 v1, v1 -; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0) -; GFX11-NEXT: v_ldexp_f32 v1, v1, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo -; GFX11-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0 -; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1 -; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0 -; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_exp10_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0 +; GFX11TRUE16-NEXT: v_fma_f32 v2, 0x40549a78, v0, -v1 +; GFX11TRUE16-NEXT: v_rndne_f32_e32 v3, v1 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v3 +; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0 +; GFX11TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x33979a37, v2 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3 +; GFX11TRUE16-NEXT: v_exp_f32_e32 v1, v1 +; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0) +; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v2 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo +; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0 +; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_exp10_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0 +; GFX11FAKE16-NEXT: v_rndne_f32_e32 v2, v1 +; GFX11FAKE16-NEXT: v_fma_f32 v3, 0x40549a78, v0, -v1 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11FAKE16-NEXT: v_sub_f32_e32 v1, v1, v2 +; GFX11FAKE16-NEXT: v_fmamk_f32 v3, v0, 0x33979a37, v3 +; GFX11FAKE16-NEXT: v_cvt_i32_f32_e32 v2, v2 +; GFX11FAKE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_add_f32_e32 v1, v1, v3 +; GFX11FAKE16-NEXT: v_exp_f32_e32 v1, v1 +; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0) +; GFX11FAKE16-NEXT: v_ldexp_f32 v1, v1, v2 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo +; GFX11FAKE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0 +; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX1250TRUE16-LABEL: v_exp10_bf16: ; GFX1250TRUE16: ; %bb.0: @@ -30943,42 +31040,23 @@ ; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; -; GFX1250TRUE16-LABEL: v_round_bf16: -; GFX1250TRUE16: ; %bb.0: -; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX1250TRUE16-NEXT: v_trunc_f32_e32 v2, v1 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v0, -v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1] -; GFX1250TRUE16-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0 -; GFX1250TRUE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250TRUE16-NEXT: v_add_f32_e32 v0, v2, v0 -; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 -; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31] -; -; GFX1250FAKE16-LABEL: v_round_bf16: -; GFX1250FAKE16: ; %bb.0: -; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250FAKE16-NEXT: v_trunc_f32_e32 v2, v1 -; GFX1250FAKE16-NEXT: v_fma_mix_f32_bf16 v0, -v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1] -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250FAKE16-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5 -; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250FAKE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1 -; GFX1250FAKE16-NEXT: v_add_f32_e32 v0, v2, v0 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 -; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31] +; GFX1250-LABEL: v_round_bf16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_trunc_f32_e32 v2, v1 +; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, -v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5 +; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1 +; GFX1250-NEXT: v_add_f32_e32 v0, v2, v0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %op = call bfloat @llvm.round.bf16(bfloat %a) ret bfloat %op } @@ -34785,39 +34863,22 @@ ; GFX11FAKE16-NEXT: v_alignbit_b32 v1, s0, v1, 16 ; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX1250TRUE16-LABEL: v_sitofp_v3i16_to_v3bf16: -; GFX1250TRUE16: ; %bb.0: -; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16 -; GFX1250TRUE16-NEXT: v_ashrrev_i32_e32 v2, 16, v0 -; GFX1250TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1 -; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v2 -; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31] -; -; GFX1250FAKE16-LABEL: v_sitofp_v3i16_to_v3bf16: -; GFX1250FAKE16: ; %bb.0: -; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250FAKE16-NEXT: v_ashrrev_i32_e32 v2, 16, v0 -; GFX1250FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16 -; GFX1250FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 16 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v2, v2 -; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v1, v1 -; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v2 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 -; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31] +; GFX1250-LABEL: v_sitofp_v3i16_to_v3bf16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_ashrrev_i32_e32 v2, 16, v0 +; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX1250-NEXT: v_bfe_i32 v1, v1, 0, 16 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_cvt_f32_i32_e32 v2, v2 +; GFX1250-NEXT: v_cvt_f32_i32_e32 v0, v0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_cvt_f32_i32_e32 v1, v1 +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %op = sitofp <3 x i16> %x to <3 x bfloat> ret <3 x bfloat> %op } @@ -35478,31 +35539,17 @@ ; GFX11FAKE16-NEXT: v_alignbit_b32 v1, s0, v2, 16 ; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX1250TRUE16-LABEL: v_sitofp_v3i32_to_v3bf16: -; GFX1250TRUE16: ; %bb.0: -; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2 -; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1 -; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0 -; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31] -; -; GFX1250FAKE16-LABEL: v_sitofp_v3i32_to_v3bf16: -; GFX1250FAKE16: ; %bb.0: -; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v1, v1 -; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v2, v2 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 -; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v1, v2, s0 -; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31] +; GFX1250-LABEL: v_sitofp_v3i32_to_v3bf16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_cvt_f32_i32_e32 v1, v1 +; GFX1250-NEXT: v_cvt_f32_i32_e32 v0, v0 +; GFX1250-NEXT: v_cvt_f32_i32_e32 v2, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v2, s0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %op = sitofp <3 x i32> %x to <3 x bfloat> ret <3 x bfloat> %op } @@ -35891,49 +35938,27 @@ ; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] ; -; GFX1250TRUE16-LABEL: v_sitofp_i64_to_bf16: -; GFX1250TRUE16: ; %bb.0: -; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250TRUE16-NEXT: v_cls_i32_e32 v3, v1 -; GFX1250TRUE16-NEXT: v_xor_b32_e32 v2, v0, v1 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250TRUE16-NEXT: v_dual_add_nc_u32 v3, -1, v3 :: v_dual_ashrrev_i32 v2, 31, v2 -; GFX1250TRUE16-NEXT: v_add_nc_u32_e32 v2, 32, v2 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250TRUE16-NEXT: v_min_u32_e32 v2, v3, v2 -; GFX1250TRUE16-NEXT: v_lshlrev_b64_e32 v[0:1], v2, v[0:1] -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250TRUE16-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX1250TRUE16-NEXT: v_dual_sub_nc_u32 v1, 32, v2 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX1250TRUE16-NEXT: v_ldexp_f32 v0, v0, v1 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 -; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31] -; -; GFX1250FAKE16-LABEL: v_sitofp_i64_to_bf16: -; GFX1250FAKE16: ; %bb.0: -; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250FAKE16-NEXT: v_cls_i32_e32 v3, v1 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250FAKE16-NEXT: v_dual_add_nc_u32 v3, -1, v3 :: v_dual_bitop2_b32 v2, v0, v1 bitop3:0x14 -; GFX1250FAKE16-NEXT: v_ashrrev_i32_e32 v2, 31, v2 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250FAKE16-NEXT: v_add_nc_u32_e32 v2, 32, v2 -; GFX1250FAKE16-NEXT: v_min_u32_e32 v2, v3, v2 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250FAKE16-NEXT: v_lshlrev_b64_e32 v[0:1], v2, v[0:1] -; GFX1250FAKE16-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250FAKE16-NEXT: v_dual_sub_nc_u32 v1, 32, v2 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54 -; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250FAKE16-NEXT: v_ldexp_f32 v0, v0, v1 -; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 -; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31] +; GFX1250-LABEL: v_sitofp_i64_to_bf16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_cls_i32_e32 v3, v1 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_dual_add_nc_u32 v3, -1, v3 :: v_dual_bitop2_b32 v2, v0, v1 bitop3:0x14 +; GFX1250-NEXT: v_ashrrev_i32_e32 v2, 31, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_add_nc_u32_e32 v2, 32, v2 +; GFX1250-NEXT: v_min_u32_e32 v2, v3, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_lshlrev_b64_e32 v[0:1], v2, v[0:1] +; GFX1250-NEXT: v_min_u32_e32 v0, 1, v0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_dual_sub_nc_u32 v1, 32, v2 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54 +; GFX1250-NEXT: v_cvt_f32_i32_e32 v0, v0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %op = sitofp i64 %x to bfloat ret bfloat %op } @@ -36755,105 +36780,55 @@ ; GFX11FAKE16-NEXT: v_alignbit_b32 v1, s0, v1, 16 ; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX1250TRUE16-LABEL: v_sitofp_v3i64_to_v3bf16: -; GFX1250TRUE16: ; %bb.0: -; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250TRUE16-NEXT: v_cls_i32_e32 v6, v5 -; GFX1250TRUE16-NEXT: v_xor_b32_e32 v7, v4, v5 -; GFX1250TRUE16-NEXT: v_cls_i32_e32 v10, v3 -; GFX1250TRUE16-NEXT: v_cls_i32_e32 v11, v1 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1250TRUE16-NEXT: v_dual_add_nc_u32 v6, -1, v6 :: v_dual_bitop2_b32 v9, v0, v1 bitop3:0x14 -; GFX1250TRUE16-NEXT: v_ashrrev_i32_e32 v7, 31, v7 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250TRUE16-NEXT: v_dual_ashrrev_i32 v9, 31, v9 :: v_dual_bitop2_b32 v8, v2, v3 bitop3:0x14 -; GFX1250TRUE16-NEXT: v_dual_add_nc_u32 v7, 32, v7 :: v_dual_ashrrev_i32 v8, 31, v8 -; GFX1250TRUE16-NEXT: v_dual_add_nc_u32 v10, -1, v10 :: v_dual_add_nc_u32 v11, -1, v11 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_add_nc_u32_e32 v9, 32, v9 -; GFX1250TRUE16-NEXT: v_min_u32_e32 v6, v6, v7 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250TRUE16-NEXT: v_add_nc_u32_e32 v8, 32, v8 -; GFX1250TRUE16-NEXT: v_lshlrev_b64_e32 v[4:5], v6, v[4:5] -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1250TRUE16-NEXT: v_min_u32_e32 v7, v10, v8 -; GFX1250TRUE16-NEXT: v_min_u32_e32 v8, v11, v9 -; GFX1250TRUE16-NEXT: v_lshlrev_b64_e32 v[2:3], v7, v[2:3] -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_lshlrev_b64_e32 v[0:1], v8, v[0:1] -; GFX1250TRUE16-NEXT: v_min_u32_e32 v4, 1, v4 -; GFX1250TRUE16-NEXT: v_min_u32_e32 v2, 1, v2 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX1250TRUE16-NEXT: v_or_b32_e32 v4, v5, v4 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_dual_sub_nc_u32 v5, 32, v8 :: v_dual_bitop2_b32 v2, v3, v2 bitop3:0x54 -; GFX1250TRUE16-NEXT: v_dual_sub_nc_u32 v3, 32, v6 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v4 -; GFX1250TRUE16-NEXT: v_sub_nc_u32_e32 v4, 32, v7 -; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX1250TRUE16-NEXT: v_ldexp_f32 v1, v1, v3 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_ldexp_f32 v2, v2, v4 -; GFX1250TRUE16-NEXT: v_ldexp_f32 v0, v0, v5 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 -; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v2 -; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31] -; -; GFX1250FAKE16-LABEL: v_sitofp_v3i64_to_v3bf16: -; GFX1250FAKE16: ; %bb.0: -; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250FAKE16-NEXT: v_cls_i32_e32 v6, v5 -; GFX1250FAKE16-NEXT: v_xor_b32_e32 v7, v2, v3 -; GFX1250FAKE16-NEXT: v_cls_i32_e32 v10, v3 -; GFX1250FAKE16-NEXT: v_cls_i32_e32 v11, v1 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1250FAKE16-NEXT: v_dual_add_nc_u32 v6, -1, v6 :: v_dual_bitop2_b32 v8, v4, v5 bitop3:0x14 -; GFX1250FAKE16-NEXT: v_dual_ashrrev_i32 v7, 31, v7 :: v_dual_bitop2_b32 v9, v0, v1 bitop3:0x14 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250FAKE16-NEXT: v_dual_add_nc_u32 v10, -1, v10 :: v_dual_ashrrev_i32 v8, 31, v8 -; GFX1250FAKE16-NEXT: v_dual_add_nc_u32 v11, -1, v11 :: v_dual_ashrrev_i32 v9, 31, v9 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250FAKE16-NEXT: v_dual_add_nc_u32 v7, 32, v7 :: v_dual_add_nc_u32 v8, 32, v8 -; GFX1250FAKE16-NEXT: v_add_nc_u32_e32 v9, 32, v9 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_min_u32_e32 v7, v10, v7 -; GFX1250FAKE16-NEXT: v_min_u32_e32 v6, v6, v8 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_min_u32_e32 v9, v11, v9 -; GFX1250FAKE16-NEXT: v_lshlrev_b64_e32 v[2:3], v7, v[2:3] -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_lshlrev_b64_e32 v[4:5], v6, v[4:5] -; GFX1250FAKE16-NEXT: v_lshlrev_b64_e32 v[0:1], v9, v[0:1] -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_min_u32_e32 v2, 1, v2 -; GFX1250FAKE16-NEXT: v_min_u32_e32 v4, 1, v4 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX1250FAKE16-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX1250FAKE16-NEXT: v_sub_nc_u32_e32 v6, 32, v6 -; GFX1250FAKE16-NEXT: v_dual_sub_nc_u32 v3, 32, v7 :: v_dual_bitop2_b32 v2, v3, v2 bitop3:0x54 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX1250FAKE16-NEXT: v_dual_sub_nc_u32 v4, 32, v9 :: v_dual_bitop2_b32 v1, v5, v4 bitop3:0x54 -; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v2, v2 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v1, v1 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_ldexp_f32 v2, v2, v3 -; GFX1250FAKE16-NEXT: v_ldexp_f32 v0, v0, v4 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250FAKE16-NEXT: v_ldexp_f32 v1, v1, v6 -; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v2 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 -; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31] +; GFX1250-LABEL: v_sitofp_v3i64_to_v3bf16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_cls_i32_e32 v6, v5 +; GFX1250-NEXT: v_xor_b32_e32 v7, v2, v3 +; GFX1250-NEXT: v_cls_i32_e32 v10, v3 +; GFX1250-NEXT: v_cls_i32_e32 v11, v1 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1250-NEXT: v_dual_add_nc_u32 v6, -1, v6 :: v_dual_bitop2_b32 v8, v4, v5 bitop3:0x14 +; GFX1250-NEXT: v_dual_ashrrev_i32 v7, 31, v7 :: v_dual_bitop2_b32 v9, v0, v1 bitop3:0x14 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_dual_add_nc_u32 v10, -1, v10 :: v_dual_ashrrev_i32 v8, 31, v8 +; GFX1250-NEXT: v_dual_add_nc_u32 v11, -1, v11 :: v_dual_ashrrev_i32 v9, 31, v9 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_dual_add_nc_u32 v7, 32, v7 :: v_dual_add_nc_u32 v8, 32, v8 +; GFX1250-NEXT: v_add_nc_u32_e32 v9, 32, v9 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_min_u32_e32 v7, v10, v7 +; GFX1250-NEXT: v_min_u32_e32 v6, v6, v8 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_min_u32_e32 v9, v11, v9 +; GFX1250-NEXT: v_lshlrev_b64_e32 v[2:3], v7, v[2:3] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_lshlrev_b64_e32 v[4:5], v6, v[4:5] +; GFX1250-NEXT: v_lshlrev_b64_e32 v[0:1], v9, v[0:1] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_min_u32_e32 v2, 1, v2 +; GFX1250-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX1250-NEXT: v_min_u32_e32 v0, 1, v0 +; GFX1250-NEXT: v_sub_nc_u32_e32 v6, 32, v6 +; GFX1250-NEXT: v_dual_sub_nc_u32 v3, 32, v7 :: v_dual_bitop2_b32 v2, v3, v2 bitop3:0x54 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_or_b32_e32 v0, v1, v0 +; GFX1250-NEXT: v_dual_sub_nc_u32 v4, 32, v9 :: v_dual_bitop2_b32 v1, v5, v4 bitop3:0x54 +; GFX1250-NEXT: v_cvt_f32_i32_e32 v2, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_cvt_f32_i32_e32 v0, v0 +; GFX1250-NEXT: v_cvt_f32_i32_e32 v1, v1 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_ldexp_f32 v2, v2, v3 +; GFX1250-NEXT: v_ldexp_f32 v0, v0, v4 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_ldexp_f32 v1, v1, v6 +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %op = sitofp <3 x i64> %x to <3 x bfloat> ret <3 x bfloat> %op } @@ -37960,39 +37935,22 @@ ; GFX11FAKE16-NEXT: v_alignbit_b32 v1, s0, v1, 16 ; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX1250TRUE16-LABEL: v_uitofp_v3i16_to_v3bf16: -; GFX1250TRUE16: ; %bb.0: -; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0 -; GFX1250TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1 -; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v2 -; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31] -; -; GFX1250FAKE16-LABEL: v_uitofp_v3i16_to_v3bf16: -; GFX1250FAKE16: ; %bb.0: -; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0 -; GFX1250FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX1250FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_cvt_f32_u32_e32 v2, v2 -; GFX1250FAKE16-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250FAKE16-NEXT: v_cvt_f32_u32_e32 v1, v1 -; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v2 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 -; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31] +; GFX1250-LABEL: v_uitofp_v3i16_to_v3bf16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_lshrrev_b32_e32 v2, 16, v0 +; GFX1250-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff, v1 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_cvt_f32_u32_e32 v2, v2 +; GFX1250-NEXT: v_cvt_f32_u32_e32 v0, v0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_cvt_f32_u32_e32 v1, v1 +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %op = uitofp <3 x i16> %x to <3 x bfloat> ret <3 x bfloat> %op } @@ -38658,31 +38616,17 @@ ; GFX11FAKE16-NEXT: v_alignbit_b32 v1, s0, v2, 16 ; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX1250TRUE16-LABEL: v_uitofp_v3i32_to_v3bf16: -; GFX1250TRUE16: ; %bb.0: -; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2 -; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1 -; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0 -; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31] -; -; GFX1250FAKE16-LABEL: v_uitofp_v3i32_to_v3bf16: -; GFX1250FAKE16: ; %bb.0: -; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250FAKE16-NEXT: v_cvt_f32_u32_e32 v1, v1 -; GFX1250FAKE16-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GFX1250FAKE16-NEXT: v_cvt_f32_u32_e32 v2, v2 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 -; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v1, v2, s0 -; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31] +; GFX1250-LABEL: v_uitofp_v3i32_to_v3bf16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_cvt_f32_u32_e32 v1, v1 +; GFX1250-NEXT: v_cvt_f32_u32_e32 v0, v0 +; GFX1250-NEXT: v_cvt_f32_u32_e32 v2, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v2, s0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %op = uitofp <3 x i32> %x to <3 x bfloat> ret <3 x bfloat> %op } @@ -39708,84 +39652,44 @@ ; GFX11FAKE16-NEXT: v_alignbit_b32 v1, s0, v1, 16 ; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX1250TRUE16-LABEL: v_uitofp_v3i64_to_v3bf16: -; GFX1250TRUE16: ; %bb.0: -; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250TRUE16-NEXT: v_clz_i32_u32_e32 v6, v5 -; GFX1250TRUE16-NEXT: v_clz_i32_u32_e32 v7, v3 -; GFX1250TRUE16-NEXT: v_clz_i32_u32_e32 v8, v1 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_min_u32_e32 v6, 32, v6 -; GFX1250TRUE16-NEXT: v_min_u32_e32 v7, 32, v7 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_min_u32_e32 v8, 32, v8 -; GFX1250TRUE16-NEXT: v_lshlrev_b64_e32 v[4:5], v6, v[4:5] -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_lshlrev_b64_e32 v[2:3], v7, v[2:3] -; GFX1250TRUE16-NEXT: v_lshlrev_b64_e32 v[0:1], v8, v[0:1] -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_min_u32_e32 v4, 1, v4 -; GFX1250TRUE16-NEXT: v_min_u32_e32 v2, 1, v2 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX1250TRUE16-NEXT: v_or_b32_e32 v4, v5, v4 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_or_b32_e32 v2, v3, v2 -; GFX1250TRUE16-NEXT: v_dual_sub_nc_u32 v3, 32, v6 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54 -; GFX1250TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v8 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v4 -; GFX1250TRUE16-NEXT: v_sub_nc_u32_e32 v4, 32, v7 -; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2 -; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GFX1250TRUE16-NEXT: v_ldexp_f32 v1, v1, v3 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250TRUE16-NEXT: v_ldexp_f32 v2, v2, v4 -; GFX1250TRUE16-NEXT: v_ldexp_f32 v0, v0, v5 -; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 -; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v2 -; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31] -; -; GFX1250FAKE16-LABEL: v_uitofp_v3i64_to_v3bf16: -; GFX1250FAKE16: ; %bb.0: -; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250FAKE16-NEXT: v_clz_i32_u32_e32 v6, v3 -; GFX1250FAKE16-NEXT: v_clz_i32_u32_e32 v7, v1 -; GFX1250FAKE16-NEXT: v_clz_i32_u32_e32 v8, v5 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_min_u32_e32 v6, 32, v6 -; GFX1250FAKE16-NEXT: v_min_u32_e32 v7, 32, v7 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_min_u32_e32 v8, 32, v8 -; GFX1250FAKE16-NEXT: v_lshlrev_b64_e32 v[2:3], v6, v[2:3] -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_lshlrev_b64_e32 v[0:1], v7, v[0:1] -; GFX1250FAKE16-NEXT: v_lshlrev_b64_e32 v[4:5], v8, v[4:5] -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_min_u32_e32 v2, 1, v2 -; GFX1250FAKE16-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_min_u32_e32 v4, 1, v4 -; GFX1250FAKE16-NEXT: v_dual_sub_nc_u32 v8, 32, v8 :: v_dual_bitop2_b32 v2, v3, v2 bitop3:0x54 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_dual_sub_nc_u32 v3, 32, v6 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54 -; GFX1250FAKE16-NEXT: v_dual_sub_nc_u32 v4, 32, v7 :: v_dual_bitop2_b32 v1, v5, v4 bitop3:0x54 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_cvt_f32_u32_e32 v2, v2 -; GFX1250FAKE16-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_cvt_f32_u32_e32 v1, v1 -; GFX1250FAKE16-NEXT: v_ldexp_f32 v2, v2, v3 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250FAKE16-NEXT: v_ldexp_f32 v0, v0, v4 -; GFX1250FAKE16-NEXT: v_ldexp_f32 v1, v1, v8 -; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v2 -; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 -; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31] +; GFX1250-LABEL: v_uitofp_v3i64_to_v3bf16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_clz_i32_u32_e32 v6, v3 +; GFX1250-NEXT: v_clz_i32_u32_e32 v7, v1 +; GFX1250-NEXT: v_clz_i32_u32_e32 v8, v5 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_min_u32_e32 v6, 32, v6 +; GFX1250-NEXT: v_min_u32_e32 v7, 32, v7 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_min_u32_e32 v8, 32, v8 +; GFX1250-NEXT: v_lshlrev_b64_e32 v[2:3], v6, v[2:3] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_lshlrev_b64_e32 v[0:1], v7, v[0:1] +; GFX1250-NEXT: v_lshlrev_b64_e32 v[4:5], v8, v[4:5] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_min_u32_e32 v2, 1, v2 +; GFX1250-NEXT: v_min_u32_e32 v0, 1, v0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX1250-NEXT: v_dual_sub_nc_u32 v8, 32, v8 :: v_dual_bitop2_b32 v2, v3, v2 bitop3:0x54 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_dual_sub_nc_u32 v3, 32, v6 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54 +; GFX1250-NEXT: v_dual_sub_nc_u32 v4, 32, v7 :: v_dual_bitop2_b32 v1, v5, v4 bitop3:0x54 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_cvt_f32_u32_e32 v2, v2 +; GFX1250-NEXT: v_cvt_f32_u32_e32 v0, v0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_cvt_f32_u32_e32 v1, v1 +; GFX1250-NEXT: v_ldexp_f32 v2, v2, v3 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_ldexp_f32 v0, v0, v4 +; GFX1250-NEXT: v_ldexp_f32 v1, v1, v8 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v2 +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %op = uitofp <3 x i64> %x to <3 x bfloat> ret <3 x bfloat> %op }
diff --git a/llvm/test/CodeGen/AMDGPU/big_alu.ll b/llvm/test/CodeGen/AMDGPU/big_alu.ll index b3b6259..fd5d4cc 100644 --- a/llvm/test/CodeGen/AMDGPU/big_alu.ll +++ b/llvm/test/CodeGen/AMDGPU/big_alu.ll
@@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 ; RUN: llc -mtriple=r600 -mcpu=cedar < %s ; This test ensures that R600 backend can handle ifcvt properly
diff --git a/llvm/test/CodeGen/AMDGPU/bswap.ll b/llvm/test/CodeGen/AMDGPU/bswap.ll index b0bb155..e2d322c 100644 --- a/llvm/test/CodeGen/AMDGPU/bswap.ll +++ b/llvm/test/CodeGen/AMDGPU/bswap.ll
@@ -571,11 +571,19 @@ ; VI-NEXT: v_perm_b32 v0, 0, v0, s4 ; VI-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_bswap_i16_zext_to_i32: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_perm_b32 v0, 0, v0, 0xc0c0001 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11-REAL16-LABEL: v_bswap_i16_zext_to_i32: +; GFX11-REAL16: ; %bb.0: +; GFX11-REAL16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-REAL16-NEXT: v_perm_b32 v0, 0, v0, 0xc0c0001 +; GFX11-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-REAL16-NEXT: v_cvt_u32_u16_e32 v0, v0.l +; GFX11-REAL16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-FAKE16-LABEL: v_bswap_i16_zext_to_i32: +; GFX11-FAKE16: ; %bb.0: +; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-FAKE16-NEXT: v_perm_b32 v0, 0, v0, 0xc0c0001 +; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %bswap = call i16 @llvm.bswap.i16(i16 %src) %zext = zext i16 %bswap to i32 ret i32 %zext
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll index fe4224f..4719d6e 100644 --- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll +++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -4966,120 +4966,63 @@ ; -------------------------------------------------------------------- define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0 -; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen -; GFX12-TRUE16-NEXT: s_not_b32 s6, s5 -; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0 -; GFX12-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v0, v0, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2 -; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB16_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4 -; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen -; GFX12-FAKE16-NEXT: s_not_b32 s6, s5 -; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0 -; GFX12-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-FAKE16-NEXT: v_add_f32_e32 v0, v0, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2 -; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB16_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: s_addk_co_i32 s16, 0x200 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_b32 s4, s16, -4 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: v_mov_b32_e32 v4, s4 +; GFX12-NEXT: s_and_b32 s4, s16, 3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_lshl_b32 s4, s4, 3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4 +; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen +; GFX12-NEXT: s_not_b32 s6, s5 +; GFX12-NEXT: s_mov_b32 s5, 0 +; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX12-NEXT: v_add_f32_e32 v0, v0, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo +; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 +; GFX12-NEXT: v_mov_b32_e32 v1, v2 +; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 +; GFX12-NEXT: s_cbranch_execnz .LBB16_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -5124,108 +5067,57 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0 -; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen -; GFX11-TRUE16-NEXT: s_not_b32 s6, s5 -; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2 -; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB16_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4 -; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3 -; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen -; GFX11-FAKE16-NEXT: s_not_b32 s6, s5 -; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2 -; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB16_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_addk_i32 s16, 0x200 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX11-NEXT: s_and_b32 s4, s16, -4 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX11-NEXT: v_mov_b32_e32 v4, s4 +; GFX11-NEXT: s_and_b32 s4, s16, 3 +; GFX11-NEXT: s_lshl_b32 s4, s4, 3 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4 +; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen +; GFX11-NEXT: s_not_b32 s6, s5 +; GFX11-NEXT: s_mov_b32 s5, 0 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-NEXT: v_add_f32_e32 v0, v0, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo +; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0 +; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 +; GFX11-NEXT: v_mov_b32_e32 v1, v2 +; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 +; GFX11-NEXT: s_cbranch_execnz .LBB16_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -5480,118 +5372,62 @@ } define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0 -; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen -; GFX12-TRUE16-NEXT: s_not_b32 s6, s5 -; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0 -; GFX12-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4 -; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB17_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4 -; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen -; GFX12-FAKE16-NEXT: s_not_b32 s6, s5 -; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0 -; GFX12-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-FAKE16-NEXT: v_add_f32_e32 v0, v0, v3 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4 -; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB17_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: s_addk_co_i32 s16, 0x200 +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_b32 s4, s16, -4 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: v_mov_b32_e32 v2, s4 +; GFX12-NEXT: s_and_b32 s4, s16, 3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_lshl_b32 s4, s4, 3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4 +; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen +; GFX12-NEXT: s_not_b32 s6, s5 +; GFX12-NEXT: s_mov_b32 s5, 0 +; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX12-NEXT: v_add_f32_e32 v0, v0, v3 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo +; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 +; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 +; GFX12-NEXT: v_mov_b32_e32 v1, v4 +; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 +; GFX12-NEXT: s_cbranch_execnz .LBB17_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -5635,106 +5471,56 @@ ; GFX942-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0 -; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen -; GFX11-TRUE16-NEXT: s_not_b32 s6, s5 -; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4 -; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB17_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4 -; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3 -; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen -; GFX11-FAKE16-NEXT: s_not_b32 s6, s5 -; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4 -; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB17_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_addk_i32 s16, 0x200 +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-NEXT: s_and_b32 s4, s16, -4 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX11-NEXT: v_mov_b32_e32 v2, s4 +; GFX11-NEXT: s_and_b32 s4, s16, 3 +; GFX11-NEXT: s_lshl_b32 s4, s4, 3 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4 +; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen +; GFX11-NEXT: s_not_b32 s6, s5 +; GFX11-NEXT: s_mov_b32 s5, 0 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-NEXT: v_add_f32_e32 v0, v0, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo +; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0 +; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 +; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 +; GFX11-NEXT: v_mov_b32_e32 v1, v4 +; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 +; GFX11-NEXT: s_cbranch_execnz .LBB17_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll index a72f68e..cecc14b 100644 --- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll +++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -4095,120 +4095,63 @@ ; -------------------------------------------------------------------- define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0 -; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen -; GFX12-TRUE16-NEXT: s_not_b32 s6, s5 -; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0 -; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2 -; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4 -; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen -; GFX12-FAKE16-NEXT: s_not_b32 s6, s5 -; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0 -; GFX12-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2 -; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB13_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: s_addk_co_i32 s16, 0x200 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_b32 s4, s16, -4 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: v_mov_b32_e32 v4, s4 +; GFX12-NEXT: s_and_b32 s4, s16, 3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_lshl_b32 s4, s4, 3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4 +; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen +; GFX12-NEXT: s_not_b32 s6, s5 +; GFX12-NEXT: s_mov_b32 s5, 0 +; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo +; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 +; GFX12-NEXT: v_mov_b32_e32 v1, v2 +; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 +; GFX12-NEXT: s_cbranch_execnz .LBB13_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -4253,108 +4196,57 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0 -; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen -; GFX11-TRUE16-NEXT: s_not_b32 s6, s5 -; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2 -; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4 -; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3 -; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen -; GFX11-FAKE16-NEXT: s_not_b32 s6, s5 -; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-FAKE16-NEXT: v_max_f32_e32 v0, v0, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2 -; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB13_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_addk_i32 s16, 0x200 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX11-NEXT: s_and_b32 s4, s16, -4 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX11-NEXT: v_mov_b32_e32 v4, s4 +; GFX11-NEXT: s_and_b32 s4, s16, 3 +; GFX11-NEXT: s_lshl_b32 s4, s4, 3 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4 +; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen +; GFX11-NEXT: s_not_b32 s6, s5 +; GFX11-NEXT: s_mov_b32 s5, 0 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-NEXT: v_max_f32_e32 v0, v0, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo +; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0 +; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 +; GFX11-NEXT: v_mov_b32_e32 v1, v2 +; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 +; GFX11-NEXT: s_cbranch_execnz .LBB13_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -4611,118 +4503,62 @@ } define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0 -; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen -; GFX12-TRUE16-NEXT: s_not_b32 s6, s5 -; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0 -; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4 -; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB14_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4 -; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen -; GFX12-FAKE16-NEXT: s_not_b32 s6, s5 -; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0 -; GFX12-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v3 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4 -; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB14_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: s_addk_co_i32 s16, 0x200 +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_b32 s4, s16, -4 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: v_mov_b32_e32 v2, s4 +; GFX12-NEXT: s_and_b32 s4, s16, 3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_lshl_b32 s4, s4, 3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4 +; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen +; GFX12-NEXT: s_not_b32 s6, s5 +; GFX12-NEXT: s_mov_b32 s5, 0 +; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v3 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo +; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 +; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 +; GFX12-NEXT: v_mov_b32_e32 v1, v4 +; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 +; GFX12-NEXT: s_cbranch_execnz .LBB14_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -4766,106 +4602,56 @@ ; GFX942-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0 -; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen -; GFX11-TRUE16-NEXT: s_not_b32 s6, s5 -; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4 -; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4 -; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3 -; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen -; GFX11-FAKE16-NEXT: s_not_b32 s6, s5 -; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-FAKE16-NEXT: v_max_f32_e32 v0, v0, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4 -; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB14_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_addk_i32 s16, 0x200 +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-NEXT: s_and_b32 s4, s16, -4 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX11-NEXT: v_mov_b32_e32 v2, s4 +; GFX11-NEXT: s_and_b32 s4, s16, 3 +; GFX11-NEXT: s_lshl_b32 s4, s4, 3 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4 +; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen +; GFX11-NEXT: s_not_b32 s6, s5 +; GFX11-NEXT: s_mov_b32 s5, 0 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-NEXT: v_max_f32_e32 v0, v0, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo +; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0 +; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 +; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 +; GFX11-NEXT: v_mov_b32_e32 v1, v4 +; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 +; GFX11-NEXT: s_cbranch_execnz .LBB14_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll index 09bbbc3..7859ac4 100644 --- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll +++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -4095,120 +4095,63 @@ ; -------------------------------------------------------------------- define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0 -; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen -; GFX12-TRUE16-NEXT: s_not_b32 s6, s5 -; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0 -; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2 -; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4 -; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen -; GFX12-FAKE16-NEXT: s_not_b32 s6, s5 -; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0 -; GFX12-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v0, v0, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2 -; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB13_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: s_addk_co_i32 s16, 0x200 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_b32 s4, s16, -4 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: v_mov_b32_e32 v4, s4 +; GFX12-NEXT: s_and_b32 s4, s16, 3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_lshl_b32 s4, s4, 3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4 +; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen +; GFX12-NEXT: s_not_b32 s6, s5 +; GFX12-NEXT: s_mov_b32 s5, 0 +; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo +; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 +; GFX12-NEXT: v_mov_b32_e32 v1, v2 +; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 +; GFX12-NEXT: s_cbranch_execnz .LBB13_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -4253,108 +4196,57 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0 -; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen -; GFX11-TRUE16-NEXT: s_not_b32 s6, s5 -; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2 -; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4 -; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3 -; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen -; GFX11-FAKE16-NEXT: s_not_b32 s6, s5 -; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-FAKE16-NEXT: v_min_f32_e32 v0, v0, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2 -; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB13_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_addk_i32 s16, 0x200 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX11-NEXT: s_and_b32 s4, s16, -4 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX11-NEXT: v_mov_b32_e32 v4, s4 +; GFX11-NEXT: s_and_b32 s4, s16, 3 +; GFX11-NEXT: s_lshl_b32 s4, s4, 3 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4 +; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen +; GFX11-NEXT: s_not_b32 s6, s5 +; GFX11-NEXT: s_mov_b32 s5, 0 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-NEXT: v_min_f32_e32 v0, v0, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo +; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0 +; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 +; GFX11-NEXT: v_mov_b32_e32 v1, v2 +; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 +; GFX11-NEXT: s_cbranch_execnz .LBB13_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -4611,118 +4503,62 @@ } define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0 -; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen -; GFX12-TRUE16-NEXT: s_not_b32 s6, s5 -; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0 -; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4 -; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB14_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4 -; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen -; GFX12-FAKE16-NEXT: s_not_b32 s6, s5 -; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0 -; GFX12-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v0, v0, v3 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4 -; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB14_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: s_addk_co_i32 s16, 0x200 +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_b32 s4, s16, -4 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: v_mov_b32_e32 v2, s4 +; GFX12-NEXT: s_and_b32 s4, s16, 3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_lshl_b32 s4, s4, 3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4 +; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen +; GFX12-NEXT: s_not_b32 s6, s5 +; GFX12-NEXT: s_mov_b32 s5, 0 +; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v3 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo +; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 +; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 +; GFX12-NEXT: v_mov_b32_e32 v1, v4 +; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 +; GFX12-NEXT: s_cbranch_execnz .LBB14_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -4766,106 +4602,56 @@ ; GFX942-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0 -; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen -; GFX11-TRUE16-NEXT: s_not_b32 s6, s5 -; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4 -; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4 -; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3 -; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen -; GFX11-FAKE16-NEXT: s_not_b32 s6, s5 -; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-FAKE16-NEXT: v_min_f32_e32 v0, v0, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0 -; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4 -; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB14_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_addk_i32 s16, 0x200 +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-NEXT: s_and_b32 s4, s16, -4 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX11-NEXT: v_mov_b32_e32 v2, s4 +; GFX11-NEXT: s_and_b32 s4, s16, 3 +; GFX11-NEXT: s_lshl_b32 s4, s4, 3 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4 +; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen +; GFX11-NEXT: s_not_b32 s6, s5 +; GFX11-NEXT: s_mov_b32 s5, 0 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-NEXT: v_min_f32_e32 v0, v0, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo +; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0 +; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 +; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 +; GFX11-NEXT: v_mov_b32_e32 v1, v4 +; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 +; GFX11-NEXT: s_cbranch_execnz .LBB14_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll index afcc3de..eb68b42 100644 --- a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll +++ b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
@@ -1330,9 +1330,9 @@ ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 ; GFX11-TRUE16-NEXT: v_cmp_le_f16_e32 vcc_lo, v0.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_le_f16_e32 vcc_lo, v3.l, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_le_f16_e64 s0, v3.l, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: test85: @@ -1386,9 +1386,9 @@ ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 ; GFX11-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v0.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v3.l, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_gt_f16_e64 s0, v3.l, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: test86: @@ -1484,9 +1484,9 @@ ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 ; GFX11-TRUE16-NEXT: v_cmp_nle_f16_e32 vcc_lo, v0.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_nle_f16_e32 vcc_lo, v3.l, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_nle_f16_e64 s0, v3.l, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: test88: @@ -1624,9 +1624,9 @@ ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 ; GFX11-TRUE16-NEXT: v_cmp_nge_f16_e32 vcc_lo, v0.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_nge_f16_e32 vcc_lo, v3.l, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_nge_f16_e64 s0, v3.l, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: test91:
diff --git a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps_nnan.ll b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps_nnan.ll index f526cc4..e5b45fa 100644 --- a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps_nnan.ll +++ b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps_nnan.ll
@@ -525,9 +525,9 @@ ; GCN-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2 ; GCN-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 ; GCN-TRUE16-NEXT: v_cmp_le_f16_e32 vcc_lo, v0.l, v2.l -; GCN-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GCN-TRUE16-NEXT: v_cmp_le_f16_e32 vcc_lo, v3.l, v1.l -; GCN-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo +; GCN-TRUE16-NEXT: v_cmp_le_f16_e64 s0, v3.l, v1.l +; GCN-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GCN-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0 ; GCN-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GCN-FAKE16-LABEL: test85: @@ -561,9 +561,9 @@ ; GCN-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2 ; GCN-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 ; GCN-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v0.l, v2.l -; GCN-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GCN-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v3.l, v1.l -; GCN-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo +; GCN-TRUE16-NEXT: v_cmp_gt_f16_e64 s0, v3.l, v1.l +; GCN-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GCN-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0 ; GCN-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GCN-FAKE16-LABEL: test86: @@ -621,9 +621,9 @@ ; GCN-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2 ; GCN-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 ; GCN-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v0.l, v2.l -; GCN-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GCN-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v3.l, v1.l -; GCN-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo +; GCN-TRUE16-NEXT: v_cmp_gt_f16_e64 s0, v3.l, v1.l +; GCN-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GCN-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0 ; GCN-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GCN-FAKE16-LABEL: test88: @@ -705,9 +705,9 @@ ; GCN-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2 ; GCN-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 ; GCN-TRUE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, v0.l, v2.l -; GCN-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GCN-TRUE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, v3.l, v1.l -; GCN-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo +; GCN-TRUE16-NEXT: v_cmp_lt_f16_e64 s0, v3.l, v1.l +; GCN-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GCN-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0 ; GCN-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GCN-FAKE16-LABEL: test91:
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll index 981126d..e6c89e2 100644 --- a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll +++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
@@ -1590,9 +1590,9 @@ ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 1, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_ldexp_f16_e32 v0.l, v0.l, v1.l +; GFX11-SDAG-TRUE16-NEXT: v_ldexp_f16_e32 v0.l, v0.l, v0.h ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-FAKE16-LABEL: fmul_select_f16_test1: @@ -2839,20 +2839,19 @@ ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x3f80 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x4000, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0x4000, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -3006,20 +3005,19 @@ ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x3f80 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x3f00, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0x3f00, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -3784,20 +3782,19 @@ ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4100 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x4000, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0x4000, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -3953,20 +3950,19 @@ ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4040 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0xc100, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0xc100, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -4121,20 +4117,19 @@ ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xc080 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x4100, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0x4100, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -4450,20 +4445,19 @@ ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xc200 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0xc180, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0xc180, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -4619,20 +4613,19 @@ ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xdb80 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0xe000, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0xe000, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -4788,20 +4781,19 @@ ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4c00 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x3480, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0x3480, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ;
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll index 8c86df6..0a6dbff 100644 --- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll +++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -11952,114 +11952,58 @@ ; -------------------------------------------------------------------- define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB46_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB46_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB46_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB46_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB46_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -12104,106 +12048,54 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB46_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB46_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB46_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB46_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB46_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -12403,119 +12295,61 @@ } define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB47_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB47_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB47_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB47_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB47_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -12562,110 +12396,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB47_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB47_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB47_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB47_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB47_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -12871,119 +12651,61 @@ } define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB48_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB48_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB48_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB48_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB48_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -13031,110 +12753,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB48_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB48_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB48_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB48_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB48_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -14040,33 +13708,32 @@ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046 +; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) @@ -14074,7 +13741,7 @@ ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB51_1 ; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: @@ -14159,41 +13826,40 @@ ; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046 +; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB51_1 ; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: @@ -14682,111 +14348,56 @@ } define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX12-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB53_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX12-FAKE16-NEXT: .LBB53_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB53_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v6, v3 +; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: v_add_f32_e32 v3, v3, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-NEXT: v_mov_b32_e32 v4, v3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB53_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -14830,103 +14441,52 @@ ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB53_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB53_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB53_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v6, v3 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: v_add_f32_e32 v3, v3, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB53_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -15121,121 +14681,62 @@ } define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB54_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB54_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: global_wb scope:SCOPE_SYS +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_SYS +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB54_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -15282,110 +14783,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB54_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB54_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll index d4f94af..21c2f1b 100644 --- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll +++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -9786,114 +9786,58 @@ ; -------------------------------------------------------------------- define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB36_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -9938,106 +9882,54 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_max_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB36_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -10238,119 +10130,61 @@ } define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB37_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -10397,110 +10231,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_max_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB37_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -10707,119 +10487,61 @@ } define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB38_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -10867,110 +10589,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_max_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB38_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -11177,111 +10845,56 @@ } define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v6, v3 +; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-NEXT: v_mov_b32_e32 v4, v3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB39_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -11325,103 +10938,52 @@ ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v2 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v6, v3 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: v_max_f32_e32 v3, v3, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB39_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -12319,33 +11881,32 @@ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046 +; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v3 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) @@ -12353,7 +11914,7 @@ ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1 ; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: @@ -12438,41 +11999,40 @@ ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046 +; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1 ; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: @@ -12963,121 +12523,62 @@ } define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: global_wb scope:SCOPE_SYS +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_SYS +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB44_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -13124,110 +12625,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_max_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB44_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll index def9fe8..dd0bae5 100644 --- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll +++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -9786,114 +9786,58 @@ ; -------------------------------------------------------------------- define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB36_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -9938,106 +9882,54 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_min_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB36_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -10238,119 +10130,61 @@ } define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB37_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -10397,110 +10231,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_min_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB37_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -10707,119 +10487,61 @@ } define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB38_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -10867,110 +10589,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_min_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB38_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -11177,111 +10845,56 @@ } define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v6, v3 +; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: v_min_num_f32_e32 v3, v3, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-NEXT: v_mov_b32_e32 v4, v3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB39_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -11325,103 +10938,52 @@ ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v2 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v6, v3 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: v_min_f32_e32 v3, v3, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB39_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -12319,33 +11881,32 @@ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046 +; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v3 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) @@ -12353,7 +11914,7 @@ ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1 ; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: @@ -12438,41 +11999,40 @@ ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046 +; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1 ; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: @@ -12963,121 +12523,62 @@ } define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: global_wb scope:SCOPE_SYS +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_SYS +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB44_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -13124,110 +12625,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_min_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB44_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll index 45c72e7..2c64aaf 100644 --- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll +++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -9383,114 +9383,58 @@ ; -------------------------------------------------------------------- define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB32_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB32_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fsub_ret_bf16: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB32_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fsub_ret_bf16: ; GFX942: ; %bb.0: @@ -9535,106 +9479,54 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB32_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB32_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fsub_ret_bf16: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB32_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fsub_ret_bf16: ; GFX10: ; %bb.0: @@ -9834,119 +9726,61 @@ } define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB33_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB33_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB33_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos: ; GFX942: ; %bb.0: @@ -9993,110 +9827,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB33_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB33_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB33_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos: ; GFX10: ; %bb.0: @@ -10302,119 +10082,61 @@ } define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB34_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB34_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg: ; GFX942: ; %bb.0: @@ -10462,110 +10184,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB34_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB34_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg: ; GFX10: ; %bb.0: @@ -10771,111 +10439,56 @@ } define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB35_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_noret_bf16: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB35_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_agent_atomic_fsub_noret_bf16: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v6, v3 +; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: v_sub_f32_e32 v3, v3, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-NEXT: v_mov_b32_e32 v4, v3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB35_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_agent_atomic_fsub_noret_bf16: ; GFX942: ; %bb.0: @@ -10919,103 +10532,52 @@ ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB35_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_noret_bf16: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB35_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_agent_atomic_fsub_noret_bf16: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v6, v3 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: v_sub_f32_e32 v3, v3, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB35_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_agent_atomic_fsub_noret_bf16: ; GFX10: ; %bb.0: @@ -11910,33 +11472,32 @@ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046 +; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) @@ -11944,7 +11505,7 @@ ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1 ; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4: @@ -12029,41 +11590,40 @@ ; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046 +; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1 ; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4: @@ -12552,121 +12112,62 @@ } define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB40_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB40_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: flat_load_b32 v5, v[0:1] +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: global_wb scope:SCOPE_SYS +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_SYS +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB40_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos: ; GFX942: ; %bb.0: @@ -12713,110 +12214,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB40_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1] -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB40_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: flat_load_b32 v5, v[0:1] +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc +; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB40_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos: ; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll index fe36142..5fac7e0 100644 --- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll +++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll
@@ -1670,12 +1670,12 @@ ; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v0.h, v2.l, v0.h ; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v2.l, 8, v0.l -; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v2.h, 0, 1, vcc_lo ; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 7, v0.h ; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, 0x80, v2.l ; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, 0, vcc_lo -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.h, v3.l +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.h, v2.h ; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, 0, s0 ; GFX1250-TRUE16-NEXT: v_cndmask_b16 v2.h, 0, 8, s0 @@ -1828,124 +1828,116 @@ ; GFX1250-TRUE16: ; %bb.0: ; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.h, v0.l +; GFX1250-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v2.l, v0.l ; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x400 -; GFX1250-TRUE16-NEXT: v_frexp_mant_f16_e32 v2.l, v0.l -; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v0 +; GFX1250-TRUE16-NEXT: v_frexp_mant_f16_e32 v1.h, v0.l +; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v0 ; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v2.h, 8, v0.l -; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v3.l, 2, v1.h -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v2.l, v1.l, 0x3ff bitop3:0xec -; GFX1250-TRUE16-NEXT: v_and_b16 v4.l, 0x3ff, v2.l -; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v5.l, 6, v2.l -; GFX1250-TRUE16-NEXT: v_min_u16 v3.l, v3.l, 15 -; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 63 -; GFX1250-TRUE16-NEXT: v_frexp_mant_f16_e32 v5.h, v12.l -; GFX1250-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v6.l, v12.l +; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v3.h, 2, v2.l +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.l, v1.h, v1.l, 0x3ff bitop3:0xec +; GFX1250-TRUE16-NEXT: v_and_b16 v4.l, 0x3ff, v1.h +; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v5.l, 6, v1.h +; GFX1250-TRUE16-NEXT: v_and_b16 v1.h, v1.h, 63 +; GFX1250-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v6.l, v11.l +; GFX1250-TRUE16-NEXT: v_min_u16 v3.h, v3.h, 15 +; GFX1250-TRUE16-NEXT: v_frexp_mant_f16_e32 v5.h, v11.l ; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v6.h, 7, v4.l -; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v7.l, v3.l, 1 clamp -; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.l +; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.h +; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v1.h, 2, v6.l +; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v7.l, v3.h, 1 clamp ; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v5.h, v1.l, 0x3ff bitop3:0xec -; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v2.l, 2, v6.l ; GFX1250-TRUE16-NEXT: v_and_b16 v7.h, 0x3ff, v5.h -; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v8.l, v7.l, 1 -; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo -; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v8.h, 6, v5.h +; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v8.l, 6, v5.h +; GFX1250-TRUE16-NEXT: v_min_u16 v1.h, v1.h, 15 +; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v8.h, v7.l, 1 ; GFX1250-TRUE16-NEXT: v_and_b16 v5.h, v5.h, 63 -; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v9.l, v3.l, v3.h -; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v3.l, v8.l, -1 -; GFX1250-TRUE16-NEXT: v_min_u16 v2.l, v2.l, 15 -; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h -; GFX1250-TRUE16-NEXT: v_and_b16 v8.l, v6.h, 1 -; GFX1250-TRUE16-NEXT: v_and_b16 v5.h, v9.l, 1 -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.l, v4.l, v3.l, 0x400 bitop3:0xc8 -; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v4.l, v2.l, 1 clamp -; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0 -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v5.l, v5.l, v10.l, v8.l bitop3:0xe0 -; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v3.h, v7.l, v3.h -; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.l -; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, v4.l, 1 -; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v9.h, 7, v7.h -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v5.l, v6.h, v5.l -; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v7.l, v2.l, v1.l -; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0 -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v3.l, v3.l, -1 -; GFX1250-TRUE16-NEXT: v_and_b16 v10.l, v9.h, 1 -; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 7, v5.l -; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v1.l, v4.l, v1.l -; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v8.l, v13.l -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.l, v7.h, v3.l, 0x400 bitop3:0xc8 +; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v9.l, v3.h, v3.l +; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.h +; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v9.h, v1.h, 1 clamp +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v8.h, v8.h, -1 +; GFX1250-TRUE16-NEXT: v_and_b16 v3.h, v6.h, 1 +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v10.l, 0, 1, vcc_lo +; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.h +; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.h, v9.h, 1 +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v4.l, v4.l, v8.h, 0x400 bitop3:0xc8 +; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v10.h, 7, v7.h +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v5.l, v10.l, v3.h bitop3:0xe0 +; GFX1250-TRUE16-NEXT: v_and_b16 v8.h, v9.l, 1 +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v5.l, v5.h, -1 +; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e64 s1, 0, v4.l +; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v3.l, v7.l, v3.l +; GFX1250-TRUE16-NEXT: v_and_b16 v5.h, v10.h, 1 +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v3.h, v6.h, v3.h +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v5.l, v7.h, v5.l, 0x400 bitop3:0xc8 +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v6.h, 0, 1, s1 +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v7.l, 0, 1, vcc_lo +; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v4.l, v1.h, v1.l +; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v3.h +; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.l +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.l, v3.l, v6.h, v8.h bitop3:0xe0 +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v5.l, v8.l, v7.l, v5.h bitop3:0xe0 +; GFX1250-TRUE16-NEXT: v_and_b16 v7.h, v4.l, 1 +; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v1.l, v9.h, v1.l +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v6.h, 0, 1, s1 +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v3.l, 0, v3.l, s0 +; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e64 s0, 0, v1.h +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v10.h, v5.l +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v5.h, 0, 1, vcc_lo +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v1.l, v6.h, v7.h bitop3:0xe0 +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, 0, vcc_lo +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v3.l, v9.l, v3.l +; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v1.h +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v2.l, v2.l, v5.h +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, v1.l, s0 ; GFX1250-TRUE16-NEXT: v_and_b16 v4.h, 0x80, v2.h -; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0 -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v3.h, v8.l, v5.h bitop3:0xe0 -; GFX1250-TRUE16-NEXT: v_and_b16 v5.h, v7.l, 1 -; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v13.l -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX1250-TRUE16-NEXT: v_cndmask_b16 v3.h, 0, v3.h, vcc_lo -; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.l, v8.h, v11.l, v10.l bitop3:0xe0 -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.h, v4.l -; GFX1250-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.l, 0, s0 -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v3.h, v9.l, v3.h -; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc_lo -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v3.l, v9.h, v3.l -; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.l -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.h, 6 -; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.h -; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v8.l, v14.l -; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s2, 7, v3.l -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.l, 3, v1.h -; GFX1250-TRUE16-NEXT: v_cndmask_b16 v2.l, v3.h, 0, s1 -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v1.l, v8.l, v5.h bitop3:0xe0 -; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v8, 0, 1, s2 -; GFX1250-TRUE16-NEXT: v_cndmask_b16 v3.h, 0, 8, s1 -; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s1, 6, v4.l -; GFX1250-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s2 -; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, v1.l, vcc_lo -; GFX1250-TRUE16-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.h -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v4.h, v2.l, v3.h bitop3:0xfe -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v4.h, v4.l, v5.l bitop3:0xfe -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v5.l, v6.l, v8.l -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.l, v7.l, v1.l -; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 31, v0 -; GFX1250-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 15, v1.h -; GFX1250-TRUE16-NEXT: v_cndmask_b16 v2.l, v3.h, v2.l, vcc_lo -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v5.l, 6 -; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 7, v1.l -; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.l, 7, v6.l -; GFX1250-TRUE16-NEXT: s_and_b32 s3, s3, s1 -; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.h, 3, v0.h -; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, 0, s0 +; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 7, v3.l +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v5.l, 0, 1, vcc_lo +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v2.l, v2.l, 6 +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.l, v4.l, v1.l +; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s1, 6, v3.h +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s0 +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v5.l, v6.l, v5.l +; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.h, 3, v2.l ; GFX1250-TRUE16-NEXT: v_cndmask_b16 v4.l, 0, 8, s0 -; GFX1250-TRUE16-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v0.h -; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 6, v3.l -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v5.l, v3.l, v5.h bitop3:0xfe -; GFX1250-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 15, v0.h +; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s2, 7, v1.l +; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 31, v0 +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v5.l, 6 +; GFX1250-TRUE16-NEXT: v_cmp_gt_i16_e64 s0, 1, v2.l +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v4.h, v3.h, v5.h bitop3:0xfe +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.l, v4.h, v3.l, v4.l bitop3:0xfe +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, 0, s2 +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v4.l, 0, 8, s2 +; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.l, 7, v6.l +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, 0, vcc_lo +; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.h, 3, v0.h +; GFX1250-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 15, v2.l +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.h, v3.l, s0 ; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v5.l, v1.l, v4.l bitop3:0xfe +; GFX1250-TRUE16-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v0.h +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v5.l, v1.h, v5.h bitop3:0xfe +; GFX1250-TRUE16-NEXT: s_and_b32 s3, s3, s1 +; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 6, v1.h +; GFX1250-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 15, v0.h ; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s2, 15, v0.h -; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x7e -; GFX1250-TRUE16-NEXT: s_and_b32 s0, s1, s0 +; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0x7e ; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v1.l, vcc_lo ; GFX1250-TRUE16-NEXT: v_or_b16 v1.l, 0x7e, v5.l -; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 15, v1.h +; GFX1250-TRUE16-NEXT: s_and_b32 s0, s1, s0 +; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 15, v2.l ; GFX1250-TRUE16-NEXT: s_or_b32 s0, s2, s0 -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v2.h, v3.l, 0x80 bitop3:0xec -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v2.h, v1.h, 0x80 bitop3:0xec ; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.l, s0 -; GFX1250-TRUE16-NEXT: v_cmp_eq_f16_e64 s0, 0, v12.l +; GFX1250-TRUE16-NEXT: v_cmp_eq_f16_e64 s0, 0, v11.l ; GFX1250-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s3 ; GFX1250-TRUE16-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l -; GFX1250-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v1.h, s1 +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v2.l, v3.l, v1.h, s1 ; GFX1250-TRUE16-NEXT: v_cmp_class_f16_e64 s1, v0.l, 0x204 ; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s0 -; GFX1250-TRUE16-NEXT: v_cmp_class_f16_e64 s0, v12.l, 0x204 +; GFX1250-TRUE16-NEXT: v_cmp_class_f16_e64 s0, v11.l, 0x204 ; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1250-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v4.h, vcc_lo -; GFX1250-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v12.l, v12.l +; GFX1250-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v11.l, v11.l ; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.l, s0 ; GFX1250-TRUE16-NEXT: v_cmp_o_f16_e64 s0, v0.l, v0.l ; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll index cc56e0f..bcc5f49 100644 --- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll +++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
@@ -1592,93 +1592,89 @@ ; GFX1200-NEXT: s_wait_samplecnt 0x0 ; GFX1200-NEXT: s_wait_bvhcnt 0x0 ; GFX1200-NEXT: s_wait_kmcnt 0x0 -; GFX1200-NEXT: v_mov_b16_e32 v1.l, v0.l +; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX1200-NEXT: v_lshrrev_b16 v0.l, 8, v0.l ; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v1 -; GFX1200-NEXT: v_and_b16 v0.l, 0x80, v0.l -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1200-NEXT: v_frexp_mant_f32_e32 v1, v5 -; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v6, v5 +; GFX1200-NEXT: v_and_b16 v0.l, 0x80, v0.l +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX1200-NEXT: v_bfe_u32 v2, v1, 16, 1 -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX1200-NEXT: v_xor_b16 v0.h, v6.l, -1 ; GFX1200-NEXT: v_or_b32_e32 v3, 0x400000, v1 ; GFX1200-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX1200-NEXT: v_add3_u32 v2, v2, v1, 0x7fff -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1200-NEXT: v_min_u16 v0.h, v0.h, 15 ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e32 v7, v2, v3, vcc_lo -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1200-NEXT: v_sub_nc_u16 v1.l, v0.h, 1 clamp -; GFX1200-NEXT: v_lshrrev_b32_e32 v3, 16, v7 +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1200-NEXT: v_cndmask_b32_e32 v4, v2, v3, vcc_lo +; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v6, v5 +; GFX1200-NEXT: v_lshrrev_b32_e32 v3, 16, v4 ; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1200-NEXT: v_lshlrev_b16 v1.h, v1.l, 1 -; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 19, v7 +; GFX1200-NEXT: v_xor_b16 v0.h, v6.l, -1 +; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 19, v4 ; GFX1200-NEXT: v_and_b16 v2.l, 0x7f, v3.l +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1200-NEXT: v_min_u16 v0.h, v0.h, 15 ; GFX1200-NEXT: v_and_b16 v2.h, v3.l, 7 -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1200-NEXT: v_add_nc_u16 v1.h, v1.h, -1 ; GFX1200-NEXT: v_or_b16 v3.l, 0x80, v2.l +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX1200-NEXT: v_sub_nc_u16 v1.l, v0.h, 1 clamp ; GFX1200-NEXT: v_lshrrev_b16 v2.l, 4, v2.l -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h -; GFX1200-NEXT: v_and_b16 v1.h, v3.l, v1.h -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1200-NEXT: v_lshrrev_b16 v4.l, v0.h, v3.l +; GFX1200-NEXT: v_lshlrev_b16 v1.h, v1.l, 1 +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1) ; GFX1200-NEXT: v_and_b16 v2.h, v2.l, 1 ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo -; GFX1200-NEXT: v_lshrrev_b16 v3.h, v0.h, v3.l +; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, 1, vcc_lo ; GFX1200-NEXT: v_lshrrev_b16 v1.l, v1.l, v3.l +; GFX1200-NEXT: v_add_nc_u16 v1.h, v1.h, -1 +; GFX1200-NEXT: v_and_b16 v1.h, v3.l, v1.h +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3) ; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.h -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1200-NEXT: v_or_b16 v1.h, v4.l, v2.h -; GFX1200-NEXT: v_and_b16 v2.h, v3.h, 1 +; GFX1200-NEXT: v_or_b16 v1.h, v3.h, v2.h +; GFX1200-NEXT: v_and_b16 v2.h, v4.l, 1 ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0.h +; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, 1, vcc_lo ; GFX1200-NEXT: v_and_b16 v1.h, v7.l, v1.h -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1200-NEXT: v_mov_b16_e32 v4.l, v8.l +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0.h +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1200-NEXT: v_or_b16 v2.h, v3.h, v2.h ; GFX1200-NEXT: v_add_nc_u16 v1.h, v2.l, v1.h ; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1200-NEXT: v_or_b16 v2.h, v4.l, v2.h -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v1.h -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1200-NEXT: v_and_b16 v0.h, v1.l, v2.h +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v1.h +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1200-NEXT: v_cndmask_b16 v0.h, 0, v0.h, vcc_lo ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 +; GFX1200-NEXT: v_cndmask_b16 v1.l, 0, 1, s0 ; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s0 ; GFX1200-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v0.h, 0, v0.h, vcc_lo -; GFX1200-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1200-NEXT: v_add_nc_u16 v0.h, v3.h, v0.h +; GFX1200-NEXT: v_add_nc_u16 v0.h, v4.l, v0.h +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1200-NEXT: v_add_nc_u16 v1.l, v6.l, v1.l -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v0.h +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX1200-NEXT: v_add_nc_u16 v1.l, v1.l, 6 ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX1200-NEXT: v_cndmask_b16 v2.l, 0, 8, vcc_lo -; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, 0, vcc_lo -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX1200-NEXT: v_lshlrev_b16 v2.h, 3, v1.l +; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, 0, vcc_lo ; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.l +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1200-NEXT: v_or_b16 v2.l, v0.l, v2.l -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1200-NEXT: v_or_b16 v2.h, v0.l, v2.h +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1200-NEXT: v_or_b16 v0.h, v2.l, v0.h -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1200-NEXT: v_or_b16 v1.l, v2.h, v1.h ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX1200-NEXT: v_cndmask_b16 v0.h, v1.l, v0.h, vcc_lo ; GFX1200-NEXT: v_cmp_o_f32_e32 vcc_lo, v5, v5 ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1200-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, s0 ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, vcc_lo ; GFX1200-NEXT: s_setpc_b64 s[30:31] ; @@ -1686,74 +1682,70 @@ ; GFX1250-TRUE16: ; %bb.0: ; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0x80 +; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x80 ; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v0.l -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1250-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v6, v5 +; GFX1250-TRUE16-NEXT: v_frexp_mant_f32_e32 v1, v5 ; GFX1250-TRUE16-NEXT: v_and_b16 v0.l, 0x80, v0.l -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1250-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v7, v6 -; GFX1250-TRUE16-NEXT: v_frexp_mant_f32_e32 v1, v6 -; GFX1250-TRUE16-NEXT: v_xor_b16 v0.h, v7.l, -1 +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-TRUE16-NEXT: v_xor_b16 v0.h, v6.l, -1 +; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v4, v1, s0 ; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 ; GFX1250-TRUE16-NEXT: v_min_u16 v0.h, v0.h, 15 -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, 0x7f, v1.l -; GFX1250-TRUE16-NEXT: v_and_b16 v2.h, v1.l, 7 -; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v1.h, v0.h, 1 clamp +; GFX1250-TRUE16-NEXT: v_and_b16 v1.h, 0x7f, v4.l +; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, v4.l, 7 ; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v4.l, 4, v2.l -; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, v1.h, 1 -; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v2.h, v3.l, -1 -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.l, v1.l, v3.h, 0x7f bitop3:0xec -; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v1.l, 3, v1.l -; GFX1250-TRUE16-NEXT: v_and_b16 v3.h, v4.l, 1 -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v2.l, v2.h, 0x80 bitop3:0xc8 -; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v2.h, v0.h, v3.l -; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v1.h, v1.h, v3.l -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v1.l, v5.l, v3.h bitop3:0xe0 +; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v1.l, v0.h, 1 clamp +; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v3.h, 4, v1.h +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.l -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, v2.h, 1 -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.l, v4.l, v1.l -; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo -; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0.h -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 7, v1.l -; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0 -; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, 0, s0 -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v1.h, v3.l, v2.l bitop3:0xe0 -; GFX1250-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l -; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, v0.h, vcc_lo -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, v2.l -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.h, v0.h -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.h, 6 -; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v0.h +; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, v1.l, 1 +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v4.h, 0, 1, vcc_lo ; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 3, v1.h +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v2.l, v2.h, -1 +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.h, v4.l, v3.l, 0x7f bitop3:0xec +; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v3.l, 3, v4.l +; GFX1250-TRUE16-NEXT: v_and_b16 v4.l, v3.h, 1 +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v1.h, v2.l, 0x80 bitop3:0xc8 +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v2.l, v0.h, v2.h +; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v1.l, v1.l, v2.h +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.l, v3.l, v4.h, v4.l bitop3:0xe0 +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.h +; GFX1250-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 1 +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v2.h, v3.h, v3.l +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v3.l, 0, 1, vcc_lo +; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0.h +; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.h +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v1.l, v3.l, v1.h bitop3:0xe0 +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0 +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.h, 0, s0 +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, v0.h, vcc_lo +; GFX1250-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.l, v6.l, v1.l +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, v0.h +; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, 6 +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v0.h +; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 3, v1.l ; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, 0, vcc_lo ; GFX1250-TRUE16-NEXT: v_cndmask_b16 v2.h, 0, 8, vcc_lo -; GFX1250-TRUE16-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.h -; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v0.l, v1.l, v2.l bitop3:0xfe -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-TRUE16-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.l +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v0.l, v1.h, v2.l bitop3:0xfe ; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v0.l, v0.h, v2.h bitop3:0xfe +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v0.h, vcc_lo -; GFX1250-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v6, v6 -; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v5, v5 ; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, s0 +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, vcc_lo ; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31] ;
diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll index 4568866..687175b 100644 --- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll +++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll
@@ -608,377 +608,365 @@ ; GFX1200-NEXT: s_wait_samplecnt 0x0 ; GFX1200-NEXT: s_wait_bvhcnt 0x0 ; GFX1200-NEXT: s_wait_kmcnt 0x0 -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.l, v2.l -; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.h, v2.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v13, 16, v2 +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.h, v2.l +; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.l, v2.l +; GFX1200-NEXT: v_lshrrev_b32_e32 v12, 16, v2 +; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v2 ; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v2.l -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.h, v1.l -; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v3.l -; GFX1200-NEXT: v_and_b16 v5.h, 0x3ff, v3.h -; GFX1200-NEXT: v_and_b16 v6.l, v3.h, 63 -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v7.l, v13.l -; GFX1200-NEXT: v_lshrrev_b16 v3.h, 6, v3.h -; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15 -; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v5.h -; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.l -; GFX1200-NEXT: v_sub_nc_u16 v9.h, 2, v7.l -; GFX1200-NEXT: v_sub_nc_u16 v8.h, v5.l, 1 clamp -; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.h, v13.l -; GFX1200-NEXT: v_and_b16 v11.l, v5.h, 1 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo -; GFX1200-NEXT: v_lshrrev_b16 v10.h, v5.l, v8.l -; GFX1200-NEXT: v_lshlrev_b16 v9.l, v8.h, 1 -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l -; GFX1200-NEXT: v_min_u16 v5.l, v9.h, 15 -; GFX1200-NEXT: v_or_b16 v11.l, v12.l, v11.l -; GFX1200-NEXT: v_and_b16 v10.l, v6.h, 63 -; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1 -; GFX1200-NEXT: v_and_b16 v6.l, 0x3ff, v6.h -; GFX1200-NEXT: v_sub_nc_u16 v11.h, v5.l, 1 clamp -; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v11.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.l -; GFX1200-NEXT: v_and_b16 v9.l, v8.l, v9.l -; GFX1200-NEXT: v_or_b16 v9.h, 0x400, v6.l -; GFX1200-NEXT: v_lshlrev_b16 v11.l, v11.h, 1 -; GFX1200-NEXT: v_add_nc_u16 v3.h, v5.h, v3.h -; GFX1200-NEXT: v_lshrrev_b16 v5.h, v8.h, v8.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v9.l -; GFX1200-NEXT: v_and_b16 v9.l, v10.h, 1 -; GFX1200-NEXT: v_lshrrev_b16 v6.l, 7, v6.l -; GFX1200-NEXT: v_lshrrev_b16 v8.l, v5.l, v9.h -; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v6.h -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0 -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v10.l -; GFX1200-NEXT: v_and_b16 v4.l, 0x80, v4.l -; GFX1200-NEXT: v_sub_nc_u16 v7.h, 2, v4.h -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX1200-NEXT: v_mov_b16_e32 v10.l, v12.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0 -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v3.h -; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v9.l -; GFX1200-NEXT: v_add_nc_u16 v9.l, v11.l, -1 -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0 -; GFX1200-NEXT: v_mov_b16_e32 v10.l, v14.l -; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s0 -; GFX1200-NEXT: v_and_b16 v5.l, v5.h, v8.h -; GFX1200-NEXT: v_and_b16 v5.h, v9.h, v9.l -; GFX1200-NEXT: v_and_b16 v8.h, v6.l, 1 -; GFX1200-NEXT: v_mov_b16_e32 v9.l, v12.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, vcc_lo -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.h -; GFX1200-NEXT: v_or_b16 v5.h, v10.l, v8.h -; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v9.l -; GFX1200-NEXT: v_and_b16 v8.h, v8.l, 1 -; GFX1200-NEXT: v_add_nc_u16 v5.l, v10.h, v5.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo -; GFX1200-NEXT: v_and_b16 v5.h, v6.h, v5.h -; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6 -; GFX1200-NEXT: v_lshrrev_b16 v6.h, v11.h, v9.h -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l -; GFX1200-NEXT: v_mov_b16_e32 v9.l, v12.l -; GFX1200-NEXT: v_add_nc_u16 v5.h, v6.l, v5.h -; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v3.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v12, 16, v1 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, 8, vcc_lo -; GFX1200-NEXT: v_or_b16 v8.h, v9.l, v8.h -; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v3.l -; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h -; GFX1200-NEXT: v_or_b16 v6.l, v4.l, v6.l -; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v8.h -; GFX1200-NEXT: v_or_b16 v8.h, v4.l, v9.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo -; GFX1200-NEXT: v_or_b16 v5.l, v6.l, v5.l -; GFX1200-NEXT: v_min_u16 v6.l, v7.h, 15 -; GFX1200-NEXT: v_or_b16 v3.h, v8.h, v3.h -; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v6.h, s1 -; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v2.l -; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, v9.l -; GFX1200-NEXT: v_sub_nc_u16 v6.h, v6.l, 1 clamp -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v5.l, s0 -; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.l, v1.l -; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.l, v3.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v2 -; GFX1200-NEXT: v_lshlrev_b16 v7.h, v6.h, 1 -; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.l, s1 -; GFX1200-NEXT: v_and_b16 v4.l, 0x3ff, v5.l -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v3.l -; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, 6 -; GFX1200-NEXT: v_add_nc_u16 v7.h, v7.h, -1 -; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l -; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v4.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v2.h, 0, 8, s0 -; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v2.l, v2.l -; GFX1200-NEXT: v_lshlrev_b16 v2.l, 3, v7.l -; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s0 -; GFX1200-NEXT: v_and_b16 v7.h, v8.l, v7.h -; GFX1200-NEXT: v_or_b16 v2.h, v8.h, v2.h -; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo -; GFX1200-NEXT: v_or_b16 v2.l, v8.h, v2.l -; GFX1200-NEXT: v_lshrrev_b16 v9.l, v6.l, v8.l -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.h -; GFX1200-NEXT: v_or_b16 v2.h, v2.h, v3.l -; GFX1200-NEXT: v_and_b16 v3.l, v5.l, 63 -; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v5.h -; GFX1200-NEXT: v_and_b16 v5.h, v9.l, 1 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo -; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v7.l -; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v4.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.l -; GFX1200-NEXT: v_lshrrev_b16 v5.l, 6, v5.l -; GFX1200-NEXT: v_or_b16 v3.l, v10.l, v5.h -; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.h, v8.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v2.h, vcc_lo -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0 -; GFX1200-NEXT: v_and_b16 v2.h, v4.l, 1 -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l -; GFX1200-NEXT: v_and_b16 v3.l, v5.h, v3.l -; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v13.l -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v12.l -; GFX1200-NEXT: v_or_b16 v2.h, v7.l, v2.h +; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v3.h +; GFX1200-NEXT: v_and_b16 v4.h, 0x3ff, v3.l +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.h, v12.l +; GFX1200-NEXT: v_and_b16 v5.h, v3.l, 63 ; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v12.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v3.l, s0 +; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15 +; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v4.h +; GFX1200-NEXT: v_sub_nc_u16 v8.h, 2, v6.h +; GFX1200-NEXT: v_lshrrev_b16 v4.h, 7, v4.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.h +; GFX1200-NEXT: v_sub_nc_u16 v8.l, v5.l, 1 clamp +; GFX1200-NEXT: v_lshrrev_b16 v10.l, v5.l, v7.h +; GFX1200-NEXT: v_min_u16 v8.h, v8.h, 15 +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.l ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v8.h, vcc_lo -; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v13.l, v13.l -; GFX1200-NEXT: v_and_b16 v2.h, v5.l, v2.h -; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v5.h -; GFX1200-NEXT: v_add_nc_u16 v3.l, v9.l, v3.l -; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v6.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, vcc_lo -; GFX1200-NEXT: v_add_nc_u16 v2.h, v4.l, v2.h -; GFX1200-NEXT: v_min_u16 v4.l, v5.l, 15 -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v3.l -; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v1.l -; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v3.h, s1 -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.h -; GFX1200-NEXT: v_sub_nc_u16 v6.h, v4.l, 1 clamp -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, vcc_lo -; GFX1200-NEXT: v_and_b16 v5.l, 0x80, v5.l -; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, vcc_lo -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, s0 -; GFX1200-NEXT: v_lshlrev_b16 v8.h, v6.h, 1 -; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s0 -; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v4.l -; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, v8.l -; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v7.h -; GFX1200-NEXT: v_add_nc_u16 v8.h, v8.h, -1 -; GFX1200-NEXT: v_or_b16 v3.l, v7.l, v3.l -; GFX1200-NEXT: v_and_b16 v7.l, v6.l, 63 -; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h -; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6 -; GFX1200-NEXT: v_and_b16 v8.h, v8.l, v8.h -; GFX1200-NEXT: v_lshrrev_b16 v9.l, v4.l, v8.l -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l +; GFX1200-NEXT: v_cndmask_b16 v10.h, 0, 1, vcc_lo +; GFX1200-NEXT: v_lshlrev_b16 v9.h, v8.l, 1 +; GFX1200-NEXT: v_lshrrev_b16 v3.l, 6, v3.l +; GFX1200-NEXT: v_sub_nc_u16 v11.l, v8.h, 1 clamp +; GFX1200-NEXT: v_and_b16 v5.h, 0x3ff, v6.l +; GFX1200-NEXT: v_and_b16 v9.l, v6.l, 63 +; GFX1200-NEXT: v_add_nc_u16 v5.l, v9.h, -1 +; GFX1200-NEXT: v_and_b16 v9.h, v4.h, 1 ; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l -; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v4.h -; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v8.l -; GFX1200-NEXT: v_and_b16 v9.h, v9.l, 1 +; GFX1200-NEXT: v_or_b16 v11.h, 0x400, v5.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.l +; GFX1200-NEXT: v_and_b16 v5.l, v7.h, v5.l +; GFX1200-NEXT: v_or_b16 v9.h, v10.h, v9.h +; GFX1200-NEXT: v_lshlrev_b16 v10.h, v11.l, 1 +; GFX1200-NEXT: v_lshrrev_b16 v7.h, v8.l, v7.h +; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.l +; GFX1200-NEXT: v_and_b16 v5.l, v10.l, 1 +; GFX1200-NEXT: v_and_b16 v3.l, v3.l, v9.h +; GFX1200-NEXT: v_add_nc_u16 v9.l, v10.h, -1 +; GFX1200-NEXT: v_lshrrev_b16 v9.h, v8.h, v11.h +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 1, s1 +; GFX1200-NEXT: v_lshrrev_b16 v10.h, v11.l, v11.h +; GFX1200-NEXT: v_add_nc_u16 v3.l, v4.h, v3.l +; GFX1200-NEXT: v_and_b16 v4.h, v11.h, v9.l +; GFX1200-NEXT: v_and_b16 v9.l, v5.h, 1 +; GFX1200-NEXT: v_or_b16 v5.l, v8.l, v5.l ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.h -; GFX1200-NEXT: v_and_b16 v8.h, v7.h, 1 -; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v0.l -; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.l, v1.l +; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 1, vcc_lo +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v3.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v4.h +; GFX1200-NEXT: v_and_b16 v4.h, v9.h, 1 +; GFX1200-NEXT: v_and_b16 v5.l, v7.h, v5.l +; GFX1200-NEXT: v_or_b16 v7.h, v8.l, v9.l ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo -; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v8.h -; GFX1200-NEXT: v_or_b16 v2.h, v7.l, v2.h -; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h -; GFX1200-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX1200-NEXT: v_or_b16 v9.h, v11.l, v9.h -; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v8.h -; GFX1200-NEXT: v_lshrrev_b32_e32 v11, 16, v0 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v3.l, vcc_lo -; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v1.l -; GFX1200-NEXT: v_and_b16 v4.h, v6.h, v9.h -; GFX1200-NEXT: v_sub_nc_u16 v6.h, 2, v8.l -; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.h, v6.l -; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v0.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v5.l, v2.h, v5.l, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 1, vcc_lo +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v9.l, 0, 1, s1 +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v2.h, v1.l +; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, s0 +; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v7.h +; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, v8.l +; GFX1200-NEXT: v_or_b16 v4.h, v9.l, v4.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.h +; GFX1200-NEXT: v_add_nc_u16 v5.l, v10.l, v5.l +; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v6.l +; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, 6 +; GFX1200-NEXT: v_and_b16 v4.h, v10.h, v4.h +; GFX1200-NEXT: v_and_b16 v4.l, 0x80, v4.l +; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v5.l +; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v5.h +; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v3.h ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) ; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, s0 -; GFX1200-NEXT: v_min_u16 v3.l, v6.h, 15 -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.l -; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v2.l -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1200-NEXT: v_add_nc_u16 v4.h, v9.l, v4.h -; GFX1200-NEXT: v_sub_nc_u16 v6.h, v3.l, 1 clamp +; GFX1200-NEXT: v_sub_nc_u16 v6.l, 2, v2.h +; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 8, s1 +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 1, s2 +; GFX1200-NEXT: v_or_b16 v7.h, v4.l, v7.h +; GFX1200-NEXT: v_add_nc_u16 v4.h, v9.h, v4.h +; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, s1 +; GFX1200-NEXT: v_or_b16 v8.l, v4.l, v8.l +; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, v8.h +; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v3.h +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h +; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v3.l +; GFX1200-NEXT: v_or_b16 v3.h, v8.l, v5.l +; GFX1200-NEXT: v_min_u16 v5.l, v6.l, 15 +; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v7.l +; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.h, 6 ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0 -; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v6.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v1 -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v4.h -; GFX1200-NEXT: v_lshlrev_b16 v2.h, v6.h, 1 -; GFX1200-NEXT: v_add_nc_u16 v1.h, v5.h, v7.l -; GFX1200-NEXT: v_or_b16 v5.h, 0x400, v7.h -; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l +; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s0 ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, vcc_lo -; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, -1 -; GFX1200-NEXT: v_add_nc_u16 v1.h, v1.h, 6 -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.h, v11.l -; GFX1200-NEXT: v_lshrrev_b16 v9.l, v3.l, v5.h -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX1200-NEXT: v_and_b16 v1.l, v5.h, v2.h -; GFX1200-NEXT: v_cndmask_b16 v2.h, v4.h, 0, vcc_lo -; GFX1200-NEXT: v_or_b16 v4.h, v8.h, v7.l -; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v1.h -; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.h, v5.h -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.l -; GFX1200-NEXT: v_cndmask_b16 v1.l, v4.l, 0, s0 -; GFX1200-NEXT: v_or_b16 v4.h, v4.h, v2.h -; GFX1200-NEXT: v_or_b16 v4.l, v8.h, v7.l -; GFX1200-NEXT: v_and_b16 v7.l, v9.l, 1 +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v3.h, vcc_lo +; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.h, v1.l +; GFX1200-NEXT: v_sub_nc_u16 v7.h, v5.l, 1 clamp +; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0 +; GFX1200-NEXT: v_lshlrev_b16 v8.l, 3, v6.l +; GFX1200-NEXT: v_or_b16 v6.h, v7.l, v6.h +; GFX1200-NEXT: v_and_b16 v8.h, 0x3ff, v3.h +; GFX1200-NEXT: v_lshlrev_b16 v9.l, v7.h, 1 +; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, s2 +; GFX1200-NEXT: v_or_b16 v8.l, v7.l, v8.l +; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l +; GFX1200-NEXT: v_or_b16 v4.h, v6.h, v4.h +; GFX1200-NEXT: v_or_b16 v6.h, 0x400, v8.h +; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1 +; GFX1200-NEXT: v_or_b16 v5.h, v8.l, v5.h +; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v6.l ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo -; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.h -; GFX1200-NEXT: v_or_b16 v2.h, v3.h, v2.l -; GFX1200-NEXT: v_or_b16 v1.l, v4.l, v1.l -; GFX1200-NEXT: v_sub_nc_u16 v4.l, 2, v9.h -; GFX1200-NEXT: v_or_b16 v1.h, v10.l, v7.l -; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v5.l, s1 -; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v7.h -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, vcc_lo -; GFX1200-NEXT: v_min_u16 v3.h, v4.l, 15 -; GFX1200-NEXT: v_and_b16 v4.l, v6.l, 63 -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l -; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.l, v11.l -; GFX1200-NEXT: v_and_b16 v1.h, v5.h, v1.h -; GFX1200-NEXT: v_sub_nc_u16 v4.h, v3.h, 1 clamp +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v4.l, vcc_lo +; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.l, v2.l +; GFX1200-NEXT: v_and_b16 v4.l, v6.h, v9.l +; GFX1200-NEXT: v_lshrrev_b32_e32 v13, 16, v1 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.l, v5.h, v4.h, s0 +; GFX1200-NEXT: v_and_b16 v4.h, v3.h, 63 +; GFX1200-NEXT: v_lshrrev_b16 v5.h, v5.l, v6.h ; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v4.l -; GFX1200-NEXT: v_and_b16 v6.h, v5.l, 1 -; GFX1200-NEXT: v_and_b16 v4.l, 0x3ff, v3.l -; GFX1200-NEXT: v_and_b16 v7.l, v3.l, 63 -; GFX1200-NEXT: v_lshlrev_b16 v5.h, v4.h, 1 +; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v8.h +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v13.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v4.h +; GFX1200-NEXT: v_and_b16 v4.h, v5.h, 1 ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0 -; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l -; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v4.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l -; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, -1 -; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v4.l -; GFX1200-NEXT: v_or_b16 v6.h, v10.l, v6.h -; GFX1200-NEXT: v_lshrrev_b16 v10.h, v3.h, v7.h -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0 -; GFX1200-NEXT: v_and_b16 v5.h, v7.h, v5.h -; GFX1200-NEXT: v_and_b16 v7.l, v4.l, 1 -; GFX1200-NEXT: v_lshrrev_b16 v3.l, 6, v3.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v1.h, vcc_lo -; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h -; GFX1200-NEXT: v_and_b16 v5.h, v6.l, v6.h -; GFX1200-NEXT: v_and_b16 v6.h, v10.h, 1 -; GFX1200-NEXT: v_lshrrev_b16 v4.h, v4.h, v7.h -; GFX1200-NEXT: v_or_b16 v6.l, v10.l, v7.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0 -; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, v5.h -; GFX1200-NEXT: v_add_nc_u16 v1.h, v9.l, v1.h -; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v3.h -; GFX1200-NEXT: v_and_b16 v3.l, v3.l, v6.l -; GFX1200-NEXT: v_mov_b16_e32 v7.l, v13.l -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l -; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v1.h +; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, 1, s0 +; GFX1200-NEXT: v_and_b16 v8.h, v4.l, 1 ; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v12.l -; GFX1200-NEXT: v_add_nc_u16 v3.l, v4.l, v3.l -; GFX1200-NEXT: v_or_b16 v5.h, v7.l, v6.h +; GFX1200-NEXT: v_cndmask_b16 v9.l, 0, 1, s1 +; GFX1200-NEXT: v_lshrrev_b16 v6.h, v7.h, v6.h +; GFX1200-NEXT: v_or_b16 v4.h, v6.l, v4.h +; GFX1200-NEXT: v_sub_nc_u16 v6.l, 2, v8.l +; GFX1200-NEXT: v_lshrrev_b16 v3.h, 6, v3.h +; GFX1200-NEXT: v_or_b16 v7.h, v9.l, v8.h +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v7.l, s0 +; GFX1200-NEXT: v_and_b16 v4.h, v6.h, v4.h +; GFX1200-NEXT: v_min_u16 v6.l, v6.l, 15 +; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.h, v13.l +; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v7.h ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s2 -; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo -; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v3.l -; GFX1200-NEXT: v_and_b16 v4.l, v4.h, v5.h -; GFX1200-NEXT: v_add_nc_u16 v4.h, v8.l, v6.l -; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, 8, s2 -; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v8.h, s0 -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s3 -; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, v4.l, s1 -; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v0.l -; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6 -; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s3 -; GFX1200-NEXT: v_add_nc_u16 v6.l, v9.h, v7.l -; GFX1200-NEXT: v_add_nc_u16 v3.h, v10.h, v3.h -; GFX1200-NEXT: v_and_b16 v4.l, 0x80, v4.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v0 +; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7f, v3.l, vcc_lo +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l +; GFX1200-NEXT: v_sub_nc_u16 v7.l, v6.l, 1 clamp ; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v12.l, v12.l -; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h +; GFX1200-NEXT: v_add_nc_u16 v3.h, v4.l, v3.h +; GFX1200-NEXT: v_and_b16 v4.l, 0x3ff, v6.h +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, vcc_lo +; GFX1200-NEXT: v_lshlrev_b16 v5.l, v7.l, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, s0 +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v3.h +; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v4.l +; GFX1200-NEXT: v_add_nc_u16 v4.h, v5.h, v4.h +; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, -1 +; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v1.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, 1, vcc_lo +; GFX1200-NEXT: v_and_b16 v10.h, v6.h, 63 +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h +; GFX1200-NEXT: v_and_b16 v5.l, v7.h, v5.l +; GFX1200-NEXT: v_and_b16 v8.h, 0x80, v8.h +; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, v5.h +; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.l, v7.h +; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.l, v7.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, 8, s0 +; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, 6 +; GFX1200-NEXT: v_and_b16 v9.l, v5.h, 1 +; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 1, s1 +; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v4.l +; GFX1200-NEXT: v_lshlrev_b16 v10.l, 3, v2.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v10.h +; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0 +; GFX1200-NEXT: v_or_b16 v9.l, v9.h, v9.l +; GFX1200-NEXT: v_or_b16 v5.l, v8.h, v5.l +; GFX1200-NEXT: v_or_b16 v7.h, v8.h, v10.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l +; GFX1200-NEXT: v_and_b16 v6.l, v4.l, 1 +; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v9.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v9.l, 0, 1, vcc_lo +; GFX1200-NEXT: v_or_b16 v4.h, v5.l, v4.h +; GFX1200-NEXT: v_or_b16 v3.h, v7.h, v3.h +; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v2.h +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.h, 0, v7.l, s0 +; GFX1200-NEXT: v_or_b16 v5.l, v9.l, v6.l +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.l, v0.l +; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v6.h +; GFX1200-NEXT: v_and_b16 v3.l, 0xff, v3.l +; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v2.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.h, vcc_lo +; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v1.l +; GFX1200-NEXT: v_add_nc_u16 v4.h, v5.h, v2.h +; GFX1200-NEXT: v_sub_nc_u16 v5.h, 2, v6.l +; GFX1200-NEXT: v_and_b16 v5.l, v6.h, v5.l +; GFX1200-NEXT: v_or_b16 v2.h, v3.l, v2.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v2.l, v3.h, v8.h, vcc_lo +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v4.h +; GFX1200-NEXT: v_min_u16 v3.l, v5.h, 15 +; GFX1200-NEXT: v_add_nc_u16 v3.h, v4.l, v5.l +; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.l, v0.l +; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v1.l, v1.l +; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v1 +; GFX1200-NEXT: v_sub_nc_u16 v5.l, v3.l, 1 clamp ; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.h -; GFX1200-NEXT: v_or_b16 v0.h, v4.l, v5.h +; GFX1200-NEXT: v_and_b16 v1.l, 0x3ff, v4.l +; GFX1200-NEXT: v_lshrrev_b32_e32 v14, 16, v0 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, 8, vcc_lo +; GFX1200-NEXT: v_lshlrev_b16 v5.h, v5.l, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 1, s1 +; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v7.l +; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v1.l +; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, vcc_lo +; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, -1 +; GFX1200-NEXT: v_add_nc_u16 v6.h, v8.l, v6.h +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v14.l +; GFX1200-NEXT: v_or_b16 v1.h, v7.l, v1.h +; GFX1200-NEXT: v_and_b16 v10.l, v4.l, 63 +; GFX1200-NEXT: v_and_b16 v5.h, v7.h, v5.h +; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, 6 +; GFX1200-NEXT: v_sub_nc_u16 v8.h, 2, v8.l +; GFX1200-NEXT: v_or_b16 v1.h, v1.h, v4.h +; GFX1200-NEXT: v_lshrrev_b16 v4.h, v3.l, v7.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.h +; GFX1200-NEXT: v_lshlrev_b16 v5.h, 3, v6.h +; GFX1200-NEXT: v_min_u16 v8.h, v8.h, 15 +; GFX1200-NEXT: v_frexp_mant_f16_e32 v10.h, v14.l +; GFX1200-NEXT: v_and_b16 v9.l, v4.h, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 1, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1 +; GFX1200-NEXT: v_sub_nc_u16 v11.l, v8.h, 1 clamp +; GFX1200-NEXT: v_or_b16 v5.h, v7.l, v5.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v10.l +; GFX1200-NEXT: v_or_b16 v9.l, v9.h, v9.l +; GFX1200-NEXT: v_and_b16 v9.h, 0x3ff, v10.h +; GFX1200-NEXT: v_lshlrev_b16 v10.l, v11.l, 1 +; GFX1200-NEXT: v_lshrrev_b16 v1.l, 7, v1.l +; GFX1200-NEXT: v_lshrrev_b16 v5.l, v5.l, v7.h +; GFX1200-NEXT: v_or_b16 v3.h, v5.h, v3.h +; GFX1200-NEXT: v_or_b16 v12.l, 0x400, v9.h +; GFX1200-NEXT: v_add_nc_u16 v10.l, v10.l, -1 +; GFX1200-NEXT: v_and_b16 v5.h, v10.h, 63 +; GFX1200-NEXT: v_and_b16 v7.h, v1.l, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v11.h, 0, 1, vcc_lo +; GFX1200-NEXT: v_and_b16 v5.l, v5.l, v9.l +; GFX1200-NEXT: v_and_b16 v9.l, v12.l, v10.l +; GFX1200-NEXT: v_lshrrev_b16 v9.h, 7, v9.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.h +; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, s0 +; GFX1200-NEXT: v_or_b16 v7.h, v11.h, v7.h +; GFX1200-NEXT: v_lshrrev_b16 v4.l, 6, v4.l +; GFX1200-NEXT: v_lshrrev_b16 v5.h, v8.h, v12.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v9.l +; GFX1200-NEXT: v_and_b16 v9.l, v9.h, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v10.l, 0, 1, vcc_lo +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l +; GFX1200-NEXT: v_and_b16 v3.l, v4.l, v7.h +; GFX1200-NEXT: v_and_b16 v4.l, v5.h, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 1, s0 +; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.l +; GFX1200-NEXT: v_lshrrev_b16 v10.l, 6, v10.h +; GFX1200-NEXT: v_add_nc_u16 v1.l, v1.l, v3.l +; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v6.h +; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v4.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, vcc_lo +; GFX1200-NEXT: v_and_b16 v4.l, v10.l, v9.l +; GFX1200-NEXT: v_lshrrev_b16 v6.h, v11.l, v12.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.h, v1.h, s0 +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v1.l +; GFX1200-NEXT: v_add_nc_u16 v3.h, v4.h, v5.l +; GFX1200-NEXT: v_add_nc_u16 v4.l, v9.h, v4.l +; GFX1200-NEXT: v_and_b16 v3.l, v6.h, v3.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.h +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, 1, vcc_lo +; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.h +; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v4.l +; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v0.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v3.l, s0 +; GFX1200-NEXT: v_add_nc_u16 v4.h, v6.l, v4.h +; GFX1200-NEXT: v_cmp_eq_f16_e64 s2, 0, v13.l +; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 1, s3 +; GFX1200-NEXT: v_and_b16 v5.l, 0x80, v5.l +; GFX1200-NEXT: v_add_nc_u16 v3.l, v5.h, v3.l +; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, 8, s1 +; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6 +; GFX1200-NEXT: v_add_nc_u16 v6.l, v8.l, v6.h +; GFX1200-NEXT: v_lshrrev_b32_e32 v8, 31, v0 +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v3.l +; GFX1200-NEXT: v_or_b16 v0.h, v5.l, v5.h +; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1 ; GFX1200-NEXT: v_add_nc_u16 v5.h, v6.l, 6 ; GFX1200-NEXT: v_lshlrev_b16 v6.l, 3, v4.h -; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v7.l ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s1 -; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1 -; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v5.h -; GFX1200-NEXT: v_or_b16 v6.l, v4.l, v6.l -; GFX1200-NEXT: v_or_b16 v0.h, v0.h, v1.h -; GFX1200-NEXT: v_or_b16 v6.h, v7.l, v6.h -; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v5.h -; GFX1200-NEXT: v_or_b16 v7.h, v7.l, v7.h -; GFX1200-NEXT: v_or_b16 v1.h, v6.l, v5.l -; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, s0 -; GFX1200-NEXT: v_or_b16 v3.h, v6.h, v3.h -; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v3.l +; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s0 +; GFX1200-NEXT: v_lshlrev_b16 v7.h, 7, v8.l +; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, 0, vcc_lo +; GFX1200-NEXT: v_lshlrev_b16 v8.l, 3, v5.h +; GFX1200-NEXT: v_or_b16 v6.l, v5.l, v6.l +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s0 +; GFX1200-NEXT: v_or_b16 v6.h, v7.h, v6.h +; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, s3 +; GFX1200-NEXT: v_or_b16 v8.l, v7.h, v8.l +; GFX1200-NEXT: v_or_b16 v0.h, v0.h, v3.h +; GFX1200-NEXT: v_or_b16 v1.l, v6.l, v1.l +; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h +; GFX1200-NEXT: v_or_b16 v3.l, v6.h, v3.l +; GFX1200-NEXT: v_or_b16 v3.h, v8.l, v4.l +; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v5.h +; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v7.l, s2 +; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v13.l, v13.l ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v0.h, v1.h, v0.h, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v0.h, v1.l, v0.h, vcc_lo ; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l -; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.l -; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v11.l, v11.l ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.l, v3.h, s1 -; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v11.l +; GFX1200-NEXT: v_cndmask_b16 v1.l, v3.h, v3.l, s0 +; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v14.l +; GFX1200-NEXT: v_cndmask_b16 v1.h, 0x7f, v1.h, s1 +; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v4.l, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, vcc_lo ; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l -; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v2.h -; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v1.l ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.h, v7.l, s1 +; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.l, v7.h, s0 +; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.h +; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v14.l, v14.l +; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v2.h ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX1200-NEXT: v_cndmask_b16 v0.h, 0x7f, v0.h, vcc_lo -; GFX1200-NEXT: v_or_b32_e32 v4, 0x7f7f0000, v3 -; GFX1200-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v1.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) ; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, s0 -; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l +; GFX1200-NEXT: v_or_b32_e32 v4, 0x7f7f0000, v3 ; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1200-NEXT: v_lshrrev_b64 v[3:4], 24, v[3:4] +; GFX1200-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l -; GFX1200-NEXT: v_mov_b16_e32 v4.l, v2.h -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l +; GFX1200-NEXT: v_lshrrev_b64 v[3:4], 24, v[3:4] +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l ; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1 +; GFX1200-NEXT: v_mov_b16_e32 v4.l, v2.h ; GFX1200-NEXT: s_setpc_b64 s[30:31] ; ; GFX1250-TRUE16-LABEL: to_fp8_v5f16: @@ -1376,378 +1364,366 @@ ; GFX1200-NEXT: s_wait_samplecnt 0x0 ; GFX1200-NEXT: s_wait_bvhcnt 0x0 ; GFX1200-NEXT: s_wait_kmcnt 0x0 -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.l, v2.l -; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.h, v2.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v13, 16, v2 +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.h, v2.l +; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.l, v2.l +; GFX1200-NEXT: v_lshrrev_b32_e32 v12, 16, v2 +; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v2 ; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v2.l -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.h, v1.l -; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v3.l -; GFX1200-NEXT: v_and_b16 v5.h, 0x3ff, v3.h -; GFX1200-NEXT: v_and_b16 v6.l, v3.h, 63 -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v7.l, v13.l -; GFX1200-NEXT: v_lshrrev_b16 v3.h, 6, v3.h -; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15 -; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v5.h -; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.l -; GFX1200-NEXT: v_sub_nc_u16 v9.h, 2, v7.l -; GFX1200-NEXT: v_sub_nc_u16 v8.h, v5.l, 1 clamp -; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.h, v13.l -; GFX1200-NEXT: v_and_b16 v11.l, v5.h, 1 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo -; GFX1200-NEXT: v_lshrrev_b16 v10.h, v5.l, v8.l -; GFX1200-NEXT: v_lshlrev_b16 v9.l, v8.h, 1 -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l -; GFX1200-NEXT: v_min_u16 v5.l, v9.h, 15 -; GFX1200-NEXT: v_or_b16 v11.l, v12.l, v11.l -; GFX1200-NEXT: v_and_b16 v10.l, v6.h, 63 -; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1 -; GFX1200-NEXT: v_and_b16 v6.l, 0x3ff, v6.h -; GFX1200-NEXT: v_sub_nc_u16 v11.h, v5.l, 1 clamp -; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v11.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.l -; GFX1200-NEXT: v_and_b16 v9.l, v8.l, v9.l -; GFX1200-NEXT: v_or_b16 v9.h, 0x400, v6.l -; GFX1200-NEXT: v_lshlrev_b16 v11.l, v11.h, 1 -; GFX1200-NEXT: v_add_nc_u16 v3.h, v5.h, v3.h -; GFX1200-NEXT: v_lshrrev_b16 v5.h, v8.h, v8.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v9.l -; GFX1200-NEXT: v_and_b16 v9.l, v10.h, 1 -; GFX1200-NEXT: v_lshrrev_b16 v6.l, 7, v6.l -; GFX1200-NEXT: v_lshrrev_b16 v8.l, v5.l, v9.h -; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v6.h -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0 -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v10.l -; GFX1200-NEXT: v_and_b16 v4.l, 0x80, v4.l -; GFX1200-NEXT: v_sub_nc_u16 v7.h, 2, v4.h -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX1200-NEXT: v_mov_b16_e32 v10.l, v12.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0 -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v3.h -; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v9.l -; GFX1200-NEXT: v_add_nc_u16 v9.l, v11.l, -1 -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0 -; GFX1200-NEXT: v_mov_b16_e32 v10.l, v14.l -; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s0 -; GFX1200-NEXT: v_and_b16 v5.l, v5.h, v8.h -; GFX1200-NEXT: v_and_b16 v5.h, v9.h, v9.l -; GFX1200-NEXT: v_and_b16 v8.h, v6.l, 1 -; GFX1200-NEXT: v_mov_b16_e32 v9.l, v12.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, vcc_lo -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.h -; GFX1200-NEXT: v_or_b16 v5.h, v10.l, v8.h -; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v9.l -; GFX1200-NEXT: v_and_b16 v8.h, v8.l, 1 -; GFX1200-NEXT: v_add_nc_u16 v5.l, v10.h, v5.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo -; GFX1200-NEXT: v_and_b16 v5.h, v6.h, v5.h -; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6 -; GFX1200-NEXT: v_lshrrev_b16 v6.h, v11.h, v9.h -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l -; GFX1200-NEXT: v_mov_b16_e32 v9.l, v12.l -; GFX1200-NEXT: v_add_nc_u16 v5.h, v6.l, v5.h -; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v3.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v12, 16, v1 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, 8, vcc_lo -; GFX1200-NEXT: v_or_b16 v8.h, v9.l, v8.h -; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v3.l -; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h -; GFX1200-NEXT: v_or_b16 v6.l, v4.l, v6.l -; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v8.h -; GFX1200-NEXT: v_or_b16 v8.h, v4.l, v9.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo -; GFX1200-NEXT: v_or_b16 v5.l, v6.l, v5.l -; GFX1200-NEXT: v_min_u16 v6.l, v7.h, 15 -; GFX1200-NEXT: v_or_b16 v3.h, v8.h, v3.h -; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v6.h, s1 -; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v2.l -; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, v9.l -; GFX1200-NEXT: v_sub_nc_u16 v6.h, v6.l, 1 clamp -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v5.l, s0 -; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.l, v1.l -; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.l, v3.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v2 -; GFX1200-NEXT: v_lshlrev_b16 v7.h, v6.h, 1 -; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.l, s1 -; GFX1200-NEXT: v_and_b16 v4.l, 0x3ff, v5.l -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v3.l -; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, 6 -; GFX1200-NEXT: v_add_nc_u16 v7.h, v7.h, -1 -; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l -; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v4.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v2.h, 0, 8, s0 -; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v2.l, v2.l -; GFX1200-NEXT: v_lshlrev_b16 v2.l, 3, v7.l -; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s0 -; GFX1200-NEXT: v_and_b16 v7.h, v8.l, v7.h -; GFX1200-NEXT: v_or_b16 v2.h, v8.h, v2.h -; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo -; GFX1200-NEXT: v_or_b16 v2.l, v8.h, v2.l -; GFX1200-NEXT: v_lshrrev_b16 v9.l, v6.l, v8.l -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.h -; GFX1200-NEXT: v_or_b16 v2.h, v2.h, v3.l -; GFX1200-NEXT: v_and_b16 v3.l, v5.l, 63 -; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v5.h -; GFX1200-NEXT: v_and_b16 v5.h, v9.l, 1 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo -; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v7.l -; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v4.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.l -; GFX1200-NEXT: v_lshrrev_b16 v5.l, 6, v5.l -; GFX1200-NEXT: v_or_b16 v3.l, v10.l, v5.h -; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.h, v8.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v2.h, vcc_lo -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0 -; GFX1200-NEXT: v_and_b16 v2.h, v4.l, 1 -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l -; GFX1200-NEXT: v_and_b16 v3.l, v5.h, v3.l -; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v13.l -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v12.l -; GFX1200-NEXT: v_or_b16 v2.h, v7.l, v2.h +; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v3.h +; GFX1200-NEXT: v_and_b16 v4.h, 0x3ff, v3.l +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.h, v12.l +; GFX1200-NEXT: v_and_b16 v5.h, v3.l, 63 ; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v12.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v3.l, s0 +; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15 +; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v4.h +; GFX1200-NEXT: v_sub_nc_u16 v8.h, 2, v6.h +; GFX1200-NEXT: v_lshrrev_b16 v4.h, 7, v4.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.h +; GFX1200-NEXT: v_sub_nc_u16 v8.l, v5.l, 1 clamp +; GFX1200-NEXT: v_lshrrev_b16 v10.l, v5.l, v7.h +; GFX1200-NEXT: v_min_u16 v8.h, v8.h, 15 +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.l ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v8.h, vcc_lo -; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v13.l, v13.l -; GFX1200-NEXT: v_and_b16 v2.h, v5.l, v2.h -; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v5.h -; GFX1200-NEXT: v_add_nc_u16 v3.l, v9.l, v3.l -; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v6.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, vcc_lo -; GFX1200-NEXT: v_add_nc_u16 v2.h, v4.l, v2.h -; GFX1200-NEXT: v_min_u16 v4.l, v5.l, 15 -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v3.l -; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v1.l -; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v3.h, s1 -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.h -; GFX1200-NEXT: v_sub_nc_u16 v6.h, v4.l, 1 clamp -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, vcc_lo -; GFX1200-NEXT: v_and_b16 v5.l, 0x80, v5.l -; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, vcc_lo -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, s0 -; GFX1200-NEXT: v_lshlrev_b16 v8.h, v6.h, 1 -; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s0 -; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v4.l -; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, v8.l -; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v7.h -; GFX1200-NEXT: v_add_nc_u16 v8.h, v8.h, -1 -; GFX1200-NEXT: v_or_b16 v3.l, v7.l, v3.l -; GFX1200-NEXT: v_and_b16 v7.l, v6.l, 63 -; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h -; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6 -; GFX1200-NEXT: v_and_b16 v8.h, v8.l, v8.h -; GFX1200-NEXT: v_lshrrev_b16 v9.l, v4.l, v8.l -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l +; GFX1200-NEXT: v_cndmask_b16 v10.h, 0, 1, vcc_lo +; GFX1200-NEXT: v_lshlrev_b16 v9.h, v8.l, 1 +; GFX1200-NEXT: v_lshrrev_b16 v3.l, 6, v3.l +; GFX1200-NEXT: v_sub_nc_u16 v11.l, v8.h, 1 clamp +; GFX1200-NEXT: v_and_b16 v5.h, 0x3ff, v6.l +; GFX1200-NEXT: v_and_b16 v9.l, v6.l, 63 +; GFX1200-NEXT: v_add_nc_u16 v5.l, v9.h, -1 +; GFX1200-NEXT: v_and_b16 v9.h, v4.h, 1 ; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l -; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v4.h -; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v8.l -; GFX1200-NEXT: v_and_b16 v9.h, v9.l, 1 +; GFX1200-NEXT: v_or_b16 v11.h, 0x400, v5.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.l +; GFX1200-NEXT: v_and_b16 v5.l, v7.h, v5.l +; GFX1200-NEXT: v_or_b16 v9.h, v10.h, v9.h +; GFX1200-NEXT: v_lshlrev_b16 v10.h, v11.l, 1 +; GFX1200-NEXT: v_lshrrev_b16 v7.h, v8.l, v7.h +; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.l +; GFX1200-NEXT: v_and_b16 v5.l, v10.l, 1 +; GFX1200-NEXT: v_and_b16 v3.l, v3.l, v9.h +; GFX1200-NEXT: v_add_nc_u16 v9.l, v10.h, -1 +; GFX1200-NEXT: v_lshrrev_b16 v9.h, v8.h, v11.h +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 1, s1 +; GFX1200-NEXT: v_lshrrev_b16 v10.h, v11.l, v11.h +; GFX1200-NEXT: v_add_nc_u16 v3.l, v4.h, v3.l +; GFX1200-NEXT: v_and_b16 v4.h, v11.h, v9.l +; GFX1200-NEXT: v_and_b16 v9.l, v5.h, 1 +; GFX1200-NEXT: v_or_b16 v5.l, v8.l, v5.l ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.h -; GFX1200-NEXT: v_and_b16 v8.h, v7.h, 1 -; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v0.l -; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.l, v1.l +; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 1, vcc_lo +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v3.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v4.h +; GFX1200-NEXT: v_and_b16 v4.h, v9.h, 1 +; GFX1200-NEXT: v_and_b16 v5.l, v7.h, v5.l +; GFX1200-NEXT: v_or_b16 v7.h, v8.l, v9.l ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo -; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v8.h -; GFX1200-NEXT: v_or_b16 v2.h, v7.l, v2.h -; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h -; GFX1200-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX1200-NEXT: v_or_b16 v9.h, v11.l, v9.h -; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v8.h -; GFX1200-NEXT: v_lshrrev_b32_e32 v11, 16, v0 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v3.l, vcc_lo -; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v1.l -; GFX1200-NEXT: v_and_b16 v4.h, v6.h, v9.h -; GFX1200-NEXT: v_sub_nc_u16 v6.h, 2, v8.l -; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.h, v6.l -; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v0.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v5.l, v2.h, v5.l, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 1, vcc_lo +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v9.l, 0, 1, s1 +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v2.h, v1.l +; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, s0 +; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v7.h +; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, v8.l +; GFX1200-NEXT: v_or_b16 v4.h, v9.l, v4.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.h +; GFX1200-NEXT: v_add_nc_u16 v5.l, v10.l, v5.l +; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v6.l +; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, 6 +; GFX1200-NEXT: v_and_b16 v4.h, v10.h, v4.h +; GFX1200-NEXT: v_and_b16 v4.l, 0x80, v4.l +; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v5.l +; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v5.h +; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v3.h ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) ; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, s0 -; GFX1200-NEXT: v_min_u16 v3.l, v6.h, 15 -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.l -; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v2.l -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1200-NEXT: v_add_nc_u16 v4.h, v9.l, v4.h -; GFX1200-NEXT: v_sub_nc_u16 v6.h, v3.l, 1 clamp +; GFX1200-NEXT: v_sub_nc_u16 v6.l, 2, v2.h +; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 8, s1 +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 1, s2 +; GFX1200-NEXT: v_or_b16 v7.h, v4.l, v7.h +; GFX1200-NEXT: v_add_nc_u16 v4.h, v9.h, v4.h +; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, s1 +; GFX1200-NEXT: v_or_b16 v8.l, v4.l, v8.l +; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, v8.h +; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v3.h +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h +; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v3.l +; GFX1200-NEXT: v_or_b16 v3.h, v8.l, v5.l +; GFX1200-NEXT: v_min_u16 v5.l, v6.l, 15 +; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v7.l +; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.h, 6 ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0 -; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v6.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v1 -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v4.h -; GFX1200-NEXT: v_lshlrev_b16 v2.h, v6.h, 1 -; GFX1200-NEXT: v_add_nc_u16 v1.h, v5.h, v7.l -; GFX1200-NEXT: v_or_b16 v5.h, 0x400, v7.h -; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l +; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s0 ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, vcc_lo -; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, -1 -; GFX1200-NEXT: v_add_nc_u16 v1.h, v1.h, 6 -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.h, v11.l -; GFX1200-NEXT: v_lshrrev_b16 v9.l, v3.l, v5.h -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX1200-NEXT: v_and_b16 v1.l, v5.h, v2.h -; GFX1200-NEXT: v_cndmask_b16 v2.h, v4.h, 0, vcc_lo -; GFX1200-NEXT: v_or_b16 v4.h, v8.h, v7.l -; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v1.h -; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.h, v5.h -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.l -; GFX1200-NEXT: v_cndmask_b16 v1.l, v4.l, 0, s0 -; GFX1200-NEXT: v_or_b16 v4.h, v4.h, v2.h -; GFX1200-NEXT: v_or_b16 v4.l, v8.h, v7.l -; GFX1200-NEXT: v_and_b16 v7.l, v9.l, 1 +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v3.h, vcc_lo +; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.h, v1.l +; GFX1200-NEXT: v_sub_nc_u16 v7.h, v5.l, 1 clamp +; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0 +; GFX1200-NEXT: v_lshlrev_b16 v8.l, 3, v6.l +; GFX1200-NEXT: v_or_b16 v6.h, v7.l, v6.h +; GFX1200-NEXT: v_and_b16 v8.h, 0x3ff, v3.h +; GFX1200-NEXT: v_lshlrev_b16 v9.l, v7.h, 1 +; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, s2 +; GFX1200-NEXT: v_or_b16 v8.l, v7.l, v8.l +; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l +; GFX1200-NEXT: v_or_b16 v4.h, v6.h, v4.h +; GFX1200-NEXT: v_or_b16 v6.h, 0x400, v8.h +; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1 +; GFX1200-NEXT: v_or_b16 v5.h, v8.l, v5.h +; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v6.l ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo -; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.h -; GFX1200-NEXT: v_or_b16 v2.h, v3.h, v2.l -; GFX1200-NEXT: v_or_b16 v1.l, v4.l, v1.l -; GFX1200-NEXT: v_sub_nc_u16 v4.l, 2, v9.h -; GFX1200-NEXT: v_or_b16 v1.h, v10.l, v7.l -; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v5.l, s1 -; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v7.h -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, vcc_lo -; GFX1200-NEXT: v_min_u16 v3.h, v4.l, 15 -; GFX1200-NEXT: v_and_b16 v4.l, v6.l, 63 -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l -; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.l, v11.l -; GFX1200-NEXT: v_and_b16 v1.h, v5.h, v1.h -; GFX1200-NEXT: v_sub_nc_u16 v4.h, v3.h, 1 clamp +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v4.l, vcc_lo +; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.l, v2.l +; GFX1200-NEXT: v_and_b16 v4.l, v6.h, v9.l +; GFX1200-NEXT: v_lshrrev_b32_e32 v13, 16, v1 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.l, v5.h, v4.h, s0 +; GFX1200-NEXT: v_and_b16 v4.h, v3.h, 63 +; GFX1200-NEXT: v_lshrrev_b16 v5.h, v5.l, v6.h ; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v4.l -; GFX1200-NEXT: v_and_b16 v6.h, v5.l, 1 -; GFX1200-NEXT: v_and_b16 v4.l, 0x3ff, v3.l -; GFX1200-NEXT: v_and_b16 v7.l, v3.l, 63 -; GFX1200-NEXT: v_lshlrev_b16 v5.h, v4.h, 1 +; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v8.h +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v13.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v4.h +; GFX1200-NEXT: v_and_b16 v4.h, v5.h, 1 ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0 -; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l -; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v4.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l -; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, -1 -; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v4.l -; GFX1200-NEXT: v_or_b16 v6.h, v10.l, v6.h -; GFX1200-NEXT: v_lshrrev_b16 v10.h, v3.h, v7.h -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0 -; GFX1200-NEXT: v_and_b16 v5.h, v7.h, v5.h -; GFX1200-NEXT: v_and_b16 v7.l, v4.l, 1 -; GFX1200-NEXT: v_lshrrev_b16 v3.l, 6, v3.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v1.h, vcc_lo -; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h -; GFX1200-NEXT: v_and_b16 v5.h, v6.l, v6.h -; GFX1200-NEXT: v_and_b16 v6.h, v10.h, 1 -; GFX1200-NEXT: v_lshrrev_b16 v4.h, v4.h, v7.h -; GFX1200-NEXT: v_or_b16 v6.l, v10.l, v7.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0 -; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, v5.h -; GFX1200-NEXT: v_add_nc_u16 v1.h, v9.l, v1.h -; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v3.h -; GFX1200-NEXT: v_and_b16 v3.l, v3.l, v6.l -; GFX1200-NEXT: v_mov_b16_e32 v7.l, v13.l -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l -; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v1.h +; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, 1, s0 +; GFX1200-NEXT: v_and_b16 v8.h, v4.l, 1 ; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v12.l -; GFX1200-NEXT: v_add_nc_u16 v3.l, v4.l, v3.l -; GFX1200-NEXT: v_or_b16 v5.h, v7.l, v6.h +; GFX1200-NEXT: v_cndmask_b16 v9.l, 0, 1, s1 +; GFX1200-NEXT: v_lshrrev_b16 v6.h, v7.h, v6.h +; GFX1200-NEXT: v_or_b16 v4.h, v6.l, v4.h +; GFX1200-NEXT: v_sub_nc_u16 v6.l, 2, v8.l +; GFX1200-NEXT: v_lshrrev_b16 v3.h, 6, v3.h +; GFX1200-NEXT: v_or_b16 v7.h, v9.l, v8.h +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v7.l, s0 +; GFX1200-NEXT: v_and_b16 v4.h, v6.h, v4.h +; GFX1200-NEXT: v_min_u16 v6.l, v6.l, 15 +; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.h, v13.l +; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v7.h ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s2 -; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo -; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v3.l -; GFX1200-NEXT: v_and_b16 v4.l, v4.h, v5.h -; GFX1200-NEXT: v_add_nc_u16 v4.h, v8.l, v6.l -; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, 8, s2 -; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v8.h, s0 -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s3 -; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, v4.l, s1 -; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v0.l -; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6 -; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s3 -; GFX1200-NEXT: v_add_nc_u16 v6.l, v9.h, v7.l -; GFX1200-NEXT: v_add_nc_u16 v3.h, v10.h, v3.h -; GFX1200-NEXT: v_and_b16 v4.l, 0x80, v4.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v0 +; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7f, v3.l, vcc_lo +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l +; GFX1200-NEXT: v_sub_nc_u16 v7.l, v6.l, 1 clamp ; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v12.l, v12.l -; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h +; GFX1200-NEXT: v_add_nc_u16 v3.h, v4.l, v3.h +; GFX1200-NEXT: v_and_b16 v4.l, 0x3ff, v6.h +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, vcc_lo +; GFX1200-NEXT: v_lshlrev_b16 v5.l, v7.l, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, s0 +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v3.h +; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v4.l +; GFX1200-NEXT: v_add_nc_u16 v4.h, v5.h, v4.h +; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, -1 +; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v1.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, 1, vcc_lo +; GFX1200-NEXT: v_and_b16 v10.h, v6.h, 63 +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h +; GFX1200-NEXT: v_and_b16 v5.l, v7.h, v5.l +; GFX1200-NEXT: v_and_b16 v8.h, 0x80, v8.h +; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, v5.h +; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.l, v7.h +; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.l, v7.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, 8, s0 +; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, 6 +; GFX1200-NEXT: v_and_b16 v9.l, v5.h, 1 +; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 1, s1 +; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v4.l +; GFX1200-NEXT: v_lshlrev_b16 v10.l, 3, v2.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v10.h +; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0 +; GFX1200-NEXT: v_or_b16 v9.l, v9.h, v9.l +; GFX1200-NEXT: v_or_b16 v5.l, v8.h, v5.l +; GFX1200-NEXT: v_or_b16 v7.h, v8.h, v10.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l +; GFX1200-NEXT: v_and_b16 v6.l, v4.l, 1 +; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v9.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v9.l, 0, 1, vcc_lo +; GFX1200-NEXT: v_or_b16 v4.h, v5.l, v4.h +; GFX1200-NEXT: v_or_b16 v3.h, v7.h, v3.h +; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v2.h +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.h, 0, v7.l, s0 +; GFX1200-NEXT: v_or_b16 v5.l, v9.l, v6.l +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.l, v0.l +; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v6.h +; GFX1200-NEXT: v_and_b16 v3.l, 0xff, v3.l +; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v2.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.h, vcc_lo +; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v1.l +; GFX1200-NEXT: v_add_nc_u16 v4.h, v5.h, v2.h +; GFX1200-NEXT: v_sub_nc_u16 v5.h, 2, v6.l +; GFX1200-NEXT: v_and_b16 v5.l, v6.h, v5.l +; GFX1200-NEXT: v_or_b16 v2.h, v3.l, v2.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v2.l, v3.h, v8.h, vcc_lo +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v4.h +; GFX1200-NEXT: v_min_u16 v3.l, v5.h, 15 +; GFX1200-NEXT: v_add_nc_u16 v3.h, v4.l, v5.l +; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.l, v0.l +; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v1.l, v1.l +; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v1 +; GFX1200-NEXT: v_sub_nc_u16 v5.l, v3.l, 1 clamp ; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.h -; GFX1200-NEXT: v_or_b16 v0.h, v4.l, v5.h +; GFX1200-NEXT: v_and_b16 v1.l, 0x3ff, v4.l +; GFX1200-NEXT: v_lshrrev_b32_e32 v14, 16, v0 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, 8, vcc_lo +; GFX1200-NEXT: v_lshlrev_b16 v5.h, v5.l, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 1, s1 +; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v7.l +; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v1.l +; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, vcc_lo +; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, -1 +; GFX1200-NEXT: v_add_nc_u16 v6.h, v8.l, v6.h +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v14.l +; GFX1200-NEXT: v_or_b16 v1.h, v7.l, v1.h +; GFX1200-NEXT: v_and_b16 v10.l, v4.l, 63 +; GFX1200-NEXT: v_and_b16 v5.h, v7.h, v5.h +; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, 6 +; GFX1200-NEXT: v_sub_nc_u16 v8.h, 2, v8.l +; GFX1200-NEXT: v_or_b16 v1.h, v1.h, v4.h +; GFX1200-NEXT: v_lshrrev_b16 v4.h, v3.l, v7.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.h +; GFX1200-NEXT: v_lshlrev_b16 v5.h, 3, v6.h +; GFX1200-NEXT: v_min_u16 v8.h, v8.h, 15 +; GFX1200-NEXT: v_frexp_mant_f16_e32 v10.h, v14.l +; GFX1200-NEXT: v_and_b16 v9.l, v4.h, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 1, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1 +; GFX1200-NEXT: v_sub_nc_u16 v11.l, v8.h, 1 clamp +; GFX1200-NEXT: v_or_b16 v5.h, v7.l, v5.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v10.l +; GFX1200-NEXT: v_or_b16 v9.l, v9.h, v9.l +; GFX1200-NEXT: v_and_b16 v9.h, 0x3ff, v10.h +; GFX1200-NEXT: v_lshlrev_b16 v10.l, v11.l, 1 +; GFX1200-NEXT: v_lshrrev_b16 v1.l, 7, v1.l +; GFX1200-NEXT: v_lshrrev_b16 v5.l, v5.l, v7.h +; GFX1200-NEXT: v_or_b16 v3.h, v5.h, v3.h +; GFX1200-NEXT: v_or_b16 v12.l, 0x400, v9.h +; GFX1200-NEXT: v_add_nc_u16 v10.l, v10.l, -1 +; GFX1200-NEXT: v_and_b16 v5.h, v10.h, 63 +; GFX1200-NEXT: v_and_b16 v7.h, v1.l, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v11.h, 0, 1, vcc_lo +; GFX1200-NEXT: v_and_b16 v5.l, v5.l, v9.l +; GFX1200-NEXT: v_and_b16 v9.l, v12.l, v10.l +; GFX1200-NEXT: v_lshrrev_b16 v9.h, 7, v9.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.h +; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, s0 +; GFX1200-NEXT: v_or_b16 v7.h, v11.h, v7.h +; GFX1200-NEXT: v_lshrrev_b16 v4.l, 6, v4.l +; GFX1200-NEXT: v_lshrrev_b16 v5.h, v8.h, v12.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v9.l +; GFX1200-NEXT: v_and_b16 v9.l, v9.h, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v10.l, 0, 1, vcc_lo +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l +; GFX1200-NEXT: v_and_b16 v3.l, v4.l, v7.h +; GFX1200-NEXT: v_and_b16 v4.l, v5.h, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 1, s0 +; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.l +; GFX1200-NEXT: v_lshrrev_b16 v10.l, 6, v10.h +; GFX1200-NEXT: v_add_nc_u16 v1.l, v1.l, v3.l +; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v6.h +; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v4.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, vcc_lo +; GFX1200-NEXT: v_and_b16 v4.l, v10.l, v9.l +; GFX1200-NEXT: v_lshrrev_b16 v6.h, v11.l, v12.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.h, v1.h, s0 +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v1.l +; GFX1200-NEXT: v_add_nc_u16 v3.h, v4.h, v5.l +; GFX1200-NEXT: v_add_nc_u16 v4.l, v9.h, v4.l +; GFX1200-NEXT: v_and_b16 v3.l, v6.h, v3.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.h +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, 1, vcc_lo +; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.h +; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v4.l +; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v0.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v3.l, s0 +; GFX1200-NEXT: v_add_nc_u16 v4.h, v6.l, v4.h +; GFX1200-NEXT: v_cmp_eq_f16_e64 s2, 0, v13.l +; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 1, s3 +; GFX1200-NEXT: v_and_b16 v5.l, 0x80, v5.l +; GFX1200-NEXT: v_add_nc_u16 v3.l, v5.h, v3.l +; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, 8, s1 +; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6 +; GFX1200-NEXT: v_add_nc_u16 v6.l, v8.l, v6.h +; GFX1200-NEXT: v_lshrrev_b32_e32 v8, 31, v0 +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v3.l +; GFX1200-NEXT: v_or_b16 v0.h, v5.l, v5.h +; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1 ; GFX1200-NEXT: v_add_nc_u16 v5.h, v6.l, 6 ; GFX1200-NEXT: v_lshlrev_b16 v6.l, 3, v4.h -; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v7.l ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s1 -; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1 -; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v5.h -; GFX1200-NEXT: v_or_b16 v6.l, v4.l, v6.l -; GFX1200-NEXT: v_or_b16 v0.h, v0.h, v1.h -; GFX1200-NEXT: v_or_b16 v6.h, v7.l, v6.h -; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v5.h -; GFX1200-NEXT: v_or_b16 v7.h, v7.l, v7.h -; GFX1200-NEXT: v_or_b16 v1.h, v6.l, v5.l -; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, s0 -; GFX1200-NEXT: v_or_b16 v3.h, v6.h, v3.h -; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v3.l +; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s0 +; GFX1200-NEXT: v_lshlrev_b16 v7.h, 7, v8.l +; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, 0, vcc_lo +; GFX1200-NEXT: v_lshlrev_b16 v8.l, 3, v5.h +; GFX1200-NEXT: v_or_b16 v6.l, v5.l, v6.l +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s0 +; GFX1200-NEXT: v_or_b16 v6.h, v7.h, v6.h +; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, s3 +; GFX1200-NEXT: v_or_b16 v8.l, v7.h, v8.l +; GFX1200-NEXT: v_or_b16 v0.h, v0.h, v3.h +; GFX1200-NEXT: v_or_b16 v1.l, v6.l, v1.l +; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h +; GFX1200-NEXT: v_or_b16 v3.l, v6.h, v3.l +; GFX1200-NEXT: v_or_b16 v3.h, v8.l, v4.l +; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v5.h +; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v7.l, s2 +; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v13.l, v13.l ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v0.h, v1.h, v0.h, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v0.h, v1.l, v0.h, vcc_lo ; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l -; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.l -; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v11.l, v11.l ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.l, v3.h, s1 -; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v11.l +; GFX1200-NEXT: v_cndmask_b16 v1.l, v3.h, v3.l, s0 +; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v14.l +; GFX1200-NEXT: v_cndmask_b16 v1.h, 0x7f, v1.h, s1 +; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v4.l, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, vcc_lo ; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l -; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v2.h -; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v1.l ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.h, v7.l, s1 +; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.l, v7.h, s0 +; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.h +; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v14.l, v14.l +; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v2.h ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX1200-NEXT: v_cndmask_b16 v0.h, 0x7f, v0.h, vcc_lo -; GFX1200-NEXT: v_or_b32_e32 v5, 0x7f7f0000, v3 -; GFX1200-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v1.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) ; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, s0 -; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l +; GFX1200-NEXT: v_or_b32_e32 v5, 0x7f7f0000, v3 ; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1200-NEXT: v_lshrrev_b64 v[3:4], 24, v[4:5] +; GFX1200-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v5, 8, v5 -; GFX1200-NEXT: v_mov_b16_e32 v4.l, v2.h -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l +; GFX1200-NEXT: v_lshrrev_b64 v[3:4], 24, v[4:5] +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l ; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1 +; GFX1200-NEXT: v_lshrrev_b32_e32 v5, 8, v5 +; GFX1200-NEXT: v_mov_b16_e32 v4.l, v2.h ; GFX1200-NEXT: s_setpc_b64 s[30:31] ; ; GFX1250-TRUE16-LABEL: to_fp8_v6f16: @@ -2255,491 +2231,477 @@ ; GFX1200-NEXT: s_wait_samplecnt 0x0 ; GFX1200-NEXT: s_wait_bvhcnt 0x0 ; GFX1200-NEXT: s_wait_kmcnt 0x0 -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.l, v3.l -; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.h, v3.l +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.h, v3.l +; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.l, v3.l ; GFX1200-NEXT: v_lshrrev_b32_e32 v14, 16, v3 -; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v3.l -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.h, v2.l -; GFX1200-NEXT: v_sub_nc_u16 v5.h, 2, v4.l -; GFX1200-NEXT: v_and_b16 v7.l, 0x3ff, v4.h -; GFX1200-NEXT: v_and_b16 v7.h, v4.h, 63 -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.l, v14.l -; GFX1200-NEXT: v_lshrrev_b16 v4.h, 6, v4.h +; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v3 +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.h, v2.l +; GFX1200-NEXT: v_sub_nc_u16 v5.h, 2, v4.h +; GFX1200-NEXT: v_and_b16 v6.l, 0x3ff, v4.l +; GFX1200-NEXT: v_and_b16 v6.h, v4.l, 63 +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v14.l +; GFX1200-NEXT: v_lshrrev_b16 v4.l, 6, v4.l ; GFX1200-NEXT: v_min_u16 v5.h, v5.h, 15 -; GFX1200-NEXT: v_or_b16 v10.l, 0x400, v7.l -; GFX1200-NEXT: v_lshrrev_b16 v7.l, 7, v7.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.h -; GFX1200-NEXT: v_sub_nc_u16 v11.l, 2, v9.l -; GFX1200-NEXT: v_sub_nc_u16 v8.h, v5.h, 1 clamp -; GFX1200-NEXT: v_lshrrev_b16 v12.h, v5.h, v10.l -; GFX1200-NEXT: v_and_b16 v12.l, v7.l, 1 +; GFX1200-NEXT: v_or_b16 v10.h, 0x400, v6.l +; GFX1200-NEXT: v_lshrrev_b16 v6.l, 7, v6.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v6.h +; GFX1200-NEXT: v_sub_nc_u16 v11.l, 2, v8.l +; GFX1200-NEXT: v_sub_nc_u16 v7.h, v5.h, 1 clamp +; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v5.h +; GFX1200-NEXT: v_and_b16 v11.h, v6.l, 1 ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0 -; GFX1200-NEXT: v_min_u16 v11.l, v11.l, 15 -; GFX1200-NEXT: v_lshlrev_b16 v10.h, v8.h, 1 -; GFX1200-NEXT: v_frexp_mant_f16_e32 v8.l, v14.l +; GFX1200-NEXT: v_cndmask_b16 v12.l, 0, 1, s1 +; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.l, v14.l +; GFX1200-NEXT: v_lshlrev_b16 v10.l, v7.h, 1 +; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v3.l +; GFX1200-NEXT: v_sub_nc_u16 v9.h, 2, v3.h +; GFX1200-NEXT: v_or_b16 v11.h, v12.l, v11.h +; GFX1200-NEXT: v_lshrrev_b16 v12.l, v5.h, v10.h +; GFX1200-NEXT: v_add_nc_u16 v10.l, v10.l, -1 +; GFX1200-NEXT: v_min_u16 v5.h, v11.l, 15 +; GFX1200-NEXT: v_and_b16 v6.h, 0x3ff, v7.l +; GFX1200-NEXT: v_and_b16 v4.l, v4.l, v11.h +; GFX1200-NEXT: v_and_b16 v12.h, v7.l, 63 +; GFX1200-NEXT: v_and_b16 v10.l, v10.h, v10.l +; GFX1200-NEXT: v_sub_nc_u16 v13.l, v5.h, 1 clamp +; GFX1200-NEXT: v_or_b16 v11.h, 0x400, v6.h +; GFX1200-NEXT: v_add_nc_u16 v4.l, v6.l, v4.l +; GFX1200-NEXT: v_lshrrev_b16 v6.l, v7.h, v10.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v10.l +; GFX1200-NEXT: v_and_b16 v10.l, v12.l, 1 +; GFX1200-NEXT: v_lshrrev_b16 v6.h, 7, v6.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s3, 0, v12.h ; GFX1200-NEXT: v_and_b16 v5.l, 0x80, v5.l -; GFX1200-NEXT: v_or_b16 v12.l, v13.l, v12.l -; GFX1200-NEXT: v_sub_nc_u16 v9.h, 2, v6.h -; GFX1200-NEXT: v_add_nc_u16 v10.h, v10.h, -1 -; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v8.l -; GFX1200-NEXT: v_and_b16 v11.h, v8.l, 63 -; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v12.l -; GFX1200-NEXT: v_lshrrev_b16 v8.l, 6, v8.l -; GFX1200-NEXT: v_and_b16 v10.h, v10.l, v10.h +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v11.l, 0, 1, s1 +; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v4.l +; GFX1200-NEXT: v_lshrrev_b16 v7.l, 6, v7.l ; GFX1200-NEXT: v_min_u16 v9.h, v9.h, 15 -; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v11.h -; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.l, v4.h -; GFX1200-NEXT: v_lshrrev_b16 v7.l, v8.h, v10.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v10.h -; GFX1200-NEXT: v_and_b16 v10.h, v12.h, 1 -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v16, 0, 1, s2 -; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v4.h -; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v2.l -; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0 -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h -; GFX1200-NEXT: v_sub_nc_u16 v5.h, v11.l, 1 clamp -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v15, 0, 1, s1 -; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s1 -; GFX1200-NEXT: v_mov_b16_e32 v12.l, v13.l -; GFX1200-NEXT: v_or_b16 v13.l, 0x400, v7.h -; GFX1200-NEXT: v_lshlrev_b16 v13.h, v5.h, 1 -; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h ; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v3.l -; GFX1200-NEXT: v_or_b16 v8.h, v12.l, v10.h -; GFX1200-NEXT: v_mov_b16_e32 v12.l, v15.l -; GFX1200-NEXT: v_add_nc_u16 v10.l, v13.h, -1 -; GFX1200-NEXT: v_and_b16 v10.h, v7.h, 1 -; GFX1200-NEXT: v_lshrrev_b16 v5.h, v5.h, v13.l -; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v8.h -; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, v12.l -; GFX1200-NEXT: v_and_b16 v10.l, v13.l, v10.l -; GFX1200-NEXT: v_lshrrev_b16 v8.h, v11.l, v13.l -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, s0 -; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, 6 -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v10.l -; GFX1200-NEXT: v_mov_b16_e32 v10.l, v16.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v11.l -; GFX1200-NEXT: v_add_nc_u16 v7.l, v12.h, v7.l -; GFX1200-NEXT: v_and_b16 v11.l, v8.h, 1 +; GFX1200-NEXT: v_or_b16 v7.h, v11.l, v10.l +; GFX1200-NEXT: v_lshlrev_b16 v10.l, v13.l, 1 ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s2 -; GFX1200-NEXT: v_lshlrev_b16 v11.h, 3, v4.l -; GFX1200-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v7.l -; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v4.l -; GFX1200-NEXT: v_or_b16 v11.l, v12.l, v11.l -; GFX1200-NEXT: v_or_b16 v11.h, v5.l, v11.h -; GFX1200-NEXT: v_and_b16 v8.l, v8.l, v10.l +; GFX1200-NEXT: v_cndmask_b16 v10.h, 0, 1, s1 +; GFX1200-NEXT: v_lshrrev_b16 v11.l, v5.h, v11.h +; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v3.l, v3.l +; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v7.h +; GFX1200-NEXT: v_add_nc_u16 v10.l, v10.l, -1 +; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, v10.h +; GFX1200-NEXT: v_cndmask_b16 v10.h, 0, 1, s3 +; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.l, v2.l +; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, v6.l, s2 +; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v5.h +; GFX1200-NEXT: v_and_b16 v5.h, v11.h, v10.l +; GFX1200-NEXT: v_and_b16 v10.l, v6.h, 1 +; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6 +; GFX1200-NEXT: v_add_nc_u16 v6.l, v12.l, v6.l +; GFX1200-NEXT: v_sub_nc_u16 v7.h, v9.h, 1 clamp +; GFX1200-NEXT: v_cmp_ne_u16_e64 s3, 0, v5.h +; GFX1200-NEXT: v_and_b16 v5.h, v11.l, 1 +; GFX1200-NEXT: v_lshlrev_b16 v12.h, 3, v4.h +; GFX1200-NEXT: v_cmp_lt_i16_e64 s4, 7, v6.l +; GFX1200-NEXT: v_or_b16 v10.l, v10.h, v10.l ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v10.h, 0, 8, s2 -; GFX1200-NEXT: v_cndmask_b16 v7.l, v7.l, 0, s2 -; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v11.l -; GFX1200-NEXT: v_or_b16 v4.l, v11.h, v4.h -; GFX1200-NEXT: v_lshrrev_b32_e32 v12, 16, v2 +; GFX1200-NEXT: v_cndmask_b16 v12.l, 0, 1, s3 +; GFX1200-NEXT: v_lshrrev_b16 v11.h, v13.l, v11.h +; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, s1 +; GFX1200-NEXT: v_cndmask_b16 v10.h, 0, 8, s4 +; GFX1200-NEXT: v_cndmask_b16 v6.l, v6.l, 0, s4 +; GFX1200-NEXT: v_or_b16 v5.h, v12.l, v5.h +; GFX1200-NEXT: v_or_b16 v12.l, v5.l, v12.h +; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v10.l ; GFX1200-NEXT: v_or_b16 v10.h, v5.l, v10.h -; GFX1200-NEXT: v_cmp_o_f16_e64 s2, v1.l, v1.l -; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, v5.h, s0 -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX1200-NEXT: v_or_b16 v4.h, v10.h, v7.l -; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.h, v8.l -; GFX1200-NEXT: v_sub_nc_u16 v8.l, v9.h, 1 clamp -; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v6.l -; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, v4.h, s1 -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v7.l -; GFX1200-NEXT: v_add_nc_u16 v4.h, v8.h, v5.h -; GFX1200-NEXT: v_lshlrev_b16 v5.h, v8.l, 1 -; GFX1200-NEXT: v_or_b16 v8.h, 0x400, v7.h +; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l +; GFX1200-NEXT: v_and_b16 v9.l, 0x3ff, v3.l +; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v4.h +; GFX1200-NEXT: v_and_b16 v4.h, v11.h, v5.h +; GFX1200-NEXT: v_or_b16 v4.l, v12.l, v4.l +; GFX1200-NEXT: v_or_b16 v5.h, v10.h, v6.l +; GFX1200-NEXT: v_lshlrev_b16 v6.l, v7.h, 1 +; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, v7.l +; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, s2 +; GFX1200-NEXT: v_or_b16 v7.l, 0x400, v9.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, v5.h, s1 +; GFX1200-NEXT: v_add_nc_u16 v5.h, v6.l, -1 +; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v6.h +; GFX1200-NEXT: v_add_nc_u16 v4.h, v11.l, v4.h +; GFX1200-NEXT: v_lshrrev_b16 v6.l, v9.h, v7.l ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo +; GFX1200-NEXT: v_and_b16 v5.l, v7.l, v5.h ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0 -; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v4.h -; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, -1 -; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v3.l, v3.l -; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h -; GFX1200-NEXT: v_mov_b16_e32 v5.l, v10.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v3 -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v3.h, v4.h, 0, s1 -; GFX1200-NEXT: v_and_b16 v4.h, v8.h, v5.h -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v4.l, 0x7f, v4.l, vcc_lo -; GFX1200-NEXT: v_add_nc_u16 v3.l, v9.l, v5.l -; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, 8, s1 -; GFX1200-NEXT: v_lshlrev_b16 v5.h, 7, v10.l -; GFX1200-NEXT: v_lshrrev_b16 v9.l, v9.h, v8.h -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h -; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6 -; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v14.l, v14.l -; GFX1200-NEXT: v_or_b16 v4.h, v5.h, v5.l -; GFX1200-NEXT: v_and_b16 v10.l, v9.l, 1 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo -; GFX1200-NEXT: v_lshlrev_b16 v5.l, 3, v3.l -; GFX1200-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX1200-NEXT: v_or_b16 v3.h, v4.h, v3.h -; GFX1200-NEXT: v_cndmask_b16 v4.h, v7.l, 0, s0 -; GFX1200-NEXT: v_and_b16 v7.l, v6.l, 63 -; GFX1200-NEXT: v_or_b16 v5.l, v5.h, v5.l -; GFX1200-NEXT: v_or_b16 v10.l, v11.l, v10.l -; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v3.l -; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l -; GFX1200-NEXT: v_lshrrev_b16 v7.l, v8.l, v8.h -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v12.l -; GFX1200-NEXT: v_or_b16 v4.h, v5.l, v4.h -; GFX1200-NEXT: v_and_b16 v5.l, v7.h, 1 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo -; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v10.l -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.h -; GFX1200-NEXT: v_sub_nc_u16 v3.l, 2, v8.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v3.h, v4.h, v3.h, s0 -; GFX1200-NEXT: v_or_b16 v5.l, v11.l, v5.l -; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v2.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, vcc_lo -; GFX1200-NEXT: v_min_u16 v3.l, v3.l, 15 -; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v14.l -; GFX1200-NEXT: v_and_b16 v4.h, v6.l, v5.l -; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v12.l -; GFX1200-NEXT: v_add_nc_u16 v5.l, v9.l, v7.l -; GFX1200-NEXT: v_sub_nc_u16 v7.l, v3.l, 1 clamp -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v5.h, vcc_lo -; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.h, v4.h -; GFX1200-NEXT: v_and_b16 v5.h, 0x3ff, v6.l -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l -; GFX1200-NEXT: v_lshlrev_b16 v7.h, v7.l, 1 -; GFX1200-NEXT: v_and_b16 v8.h, 0x80, v8.h -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h -; GFX1200-NEXT: v_or_b16 v9.l, 0x400, v5.h -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 8, vcc_lo -; GFX1200-NEXT: v_add_nc_u16 v7.h, v7.h, -1 -; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0 -; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h -; GFX1200-NEXT: v_or_b16 v9.h, v8.h, v9.h -; GFX1200-NEXT: v_and_b16 v7.h, v9.l, v7.h -; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.l, v9.l -; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, v10.l -; GFX1200-NEXT: v_lshrrev_b16 v10.l, v3.l, v9.l -; GFX1200-NEXT: v_or_b16 v5.l, v9.h, v5.l -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.h -; GFX1200-NEXT: v_and_b16 v7.h, v6.l, 63 -; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, 6 -; GFX1200-NEXT: v_and_b16 v9.h, v10.l, 1 -; GFX1200-NEXT: v_and_b16 v9.l, v5.h, 1 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.h -; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v6.h -; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0 -; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l -; GFX1200-NEXT: v_or_b16 v9.h, v11.l, v9.h -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo -; GFX1200-NEXT: v_or_b16 v7.h, v8.h, v7.h -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l -; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v3.h, s1 -; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v9.h -; GFX1200-NEXT: v_mov_b16_e32 v11.l, v13.l -; GFX1200-NEXT: v_or_b16 v4.h, v7.h, v4.h -; GFX1200-NEXT: v_lshrrev_b32_e32 v14, 16, v0 -; GFX1200-NEXT: v_lshlrev_b16 v3.h, 8, v3.h -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, vcc_lo -; GFX1200-NEXT: v_or_b16 v3.l, v11.l, v9.l -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.l, v1.l -; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v6.h -; GFX1200-NEXT: v_lshrrev_b32_e32 v11, 31, v2 -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1200-NEXT: v_and_b16 v3.l, v6.l, v3.l -; GFX1200-NEXT: v_sub_nc_u16 v6.h, 2, v9.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, v5.l, vcc_lo -; GFX1200-NEXT: v_add_nc_u16 v6.l, v10.l, v7.l -; GFX1200-NEXT: v_lshlrev_b16 v9.h, 7, v11.l -; GFX1200-NEXT: v_add_nc_u16 v3.l, v5.h, v3.l -; GFX1200-NEXT: v_min_u16 v5.l, v6.h, 15 -; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v1.l -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v6.l -; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l -; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.l -; GFX1200-NEXT: v_sub_nc_u16 v6.h, v5.l, 1 clamp -; GFX1200-NEXT: v_and_b16 v2.h, 0x3ff, v5.h -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, s0 -; GFX1200-NEXT: v_cndmask_b16 v6.l, v6.l, 0, s0 -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s1 -; GFX1200-NEXT: v_lshlrev_b16 v7.l, v6.h, 1 -; GFX1200-NEXT: v_or_b16 v10.h, 0x400, v2.h -; GFX1200-NEXT: v_or_b16 v7.h, v9.h, v7.h -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, v8.h, vcc_lo -; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, v10.l -; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, -1 -; GFX1200-NEXT: v_and_b16 v8.h, v5.h, 63 -; GFX1200-NEXT: v_or_b16 v6.l, v7.h, v6.l -; GFX1200-NEXT: v_lshrrev_b16 v7.h, v5.l, v10.h -; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, 6 -; GFX1200-NEXT: v_and_b16 v7.l, v10.h, v7.l -; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s1 -; GFX1200-NEXT: v_lshrrev_b16 v2.h, 7, v2.h -; GFX1200-NEXT: v_and_b16 v10.l, v7.h, 1 -; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v10.h -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l -; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v8.l +; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, 1, s1 +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v4.h +; GFX1200-NEXT: v_and_b16 v10.l, v6.l, 1 +; GFX1200-NEXT: v_cndmask_b16 v4.l, 0x7f, v4.l, s0 ; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.l -; GFX1200-NEXT: v_lshrrev_b16 v5.h, 6, v5.h -; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v2.l, v2.l +; GFX1200-NEXT: v_add_nc_u16 v5.l, v8.l, v5.h +; GFX1200-NEXT: v_and_b16 v8.l, v3.l, 63 ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.h -; GFX1200-NEXT: v_or_b16 v7.l, v9.h, v7.l +; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, 8, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, vcc_lo +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v10.h, 0, 1, s0 +; GFX1200-NEXT: v_lshrrev_b16 v9.l, 7, v9.l +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.l +; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, 6 +; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.h, v7.l +; GFX1200-NEXT: v_or_b16 v8.l, v10.h, v10.l +; GFX1200-NEXT: v_and_b16 v7.h, v9.l, 1 +; GFX1200-NEXT: v_lshrrev_b32_e32 v12, 16, v2 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v10.l, 0, 1, vcc_lo +; GFX1200-NEXT: v_lshlrev_b16 v10.h, 3, v5.l +; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v8.l +; GFX1200-NEXT: v_lshrrev_b16 v3.l, 6, v3.l +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v12.l +; GFX1200-NEXT: v_or_b16 v7.h, v10.l, v7.h +; GFX1200-NEXT: v_or_b16 v5.h, v8.h, v5.h +; GFX1200-NEXT: v_cndmask_b16 v6.h, v6.h, 0, s1 +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.h +; GFX1200-NEXT: v_or_b16 v9.h, v8.h, v10.h +; GFX1200-NEXT: v_sub_nc_u16 v10.l, 2, v8.l +; GFX1200-NEXT: v_and_b16 v3.l, v3.l, v7.h +; GFX1200-NEXT: v_or_b16 v4.h, v5.h, v4.h +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, v7.l, vcc_lo +; GFX1200-NEXT: v_or_b16 v6.h, v9.h, v6.h +; GFX1200-NEXT: v_min_u16 v7.l, v10.l, 15 +; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v5.l +; GFX1200-NEXT: v_add_nc_u16 v3.l, v9.l, v3.l +; GFX1200-NEXT: v_add_nc_u16 v5.l, v6.l, v5.h +; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v12.l +; GFX1200-NEXT: v_sub_nc_u16 v6.l, v7.l, 1 clamp +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v4.h, v6.h, v4.h, vcc_lo +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v3.l +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v5.l +; GFX1200-NEXT: v_and_b16 v6.h, 0x3ff, v5.h +; GFX1200-NEXT: v_lshlrev_b16 v7.h, v6.l, 1 +; GFX1200-NEXT: v_lshrrev_b16 v9.l, 8, v2.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 1, vcc_lo +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v10.l, 0, 8, s0 +; GFX1200-NEXT: v_or_b16 v10.h, 0x400, v6.h +; GFX1200-NEXT: v_add_nc_u16 v7.h, v7.h, -1 +; GFX1200-NEXT: v_and_b16 v9.l, 0x80, v9.l +; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, v9.h +; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v14.l +; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, s0 +; GFX1200-NEXT: v_and_b16 v7.h, v10.h, v7.h +; GFX1200-NEXT: v_or_b16 v9.h, v9.l, v10.l +; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, 6 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, v8.h, s1 +; GFX1200-NEXT: v_lshrrev_b16 v8.h, v7.l, v10.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.h +; GFX1200-NEXT: v_or_b16 v5.l, v9.h, v5.l +; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v3.h +; GFX1200-NEXT: v_and_b16 v9.h, v5.h, 63 +; GFX1200-NEXT: v_and_b16 v10.l, v8.h, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v11.l, 0, 1, s0 +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, vcc_lo +; GFX1200-NEXT: v_lshrrev_b16 v6.h, 7, v6.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.h +; GFX1200-NEXT: v_or_b16 v7.h, v9.l, v7.h +; GFX1200-NEXT: v_or_b16 v9.h, v11.l, v10.l +; GFX1200-NEXT: v_lshrrev_b16 v6.l, v6.l, v10.h +; GFX1200-NEXT: v_and_b16 v10.l, v6.h, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v10.h, 0, 1, vcc_lo +; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v3.l +; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v3.h +; GFX1200-NEXT: v_and_b16 v3.h, v6.l, v9.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l +; GFX1200-NEXT: v_or_b16 v6.l, v10.h, v10.l +; GFX1200-NEXT: v_lshrrev_b16 v5.h, 6, v5.h +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v5.l, vcc_lo +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.l, v1.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, v3.h, s0 +; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v14.l, v14.l +; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v6.l +; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l +; GFX1200-NEXT: v_sub_nc_u16 v6.l, 2, v5.l +; GFX1200-NEXT: v_add_nc_u16 v3.h, v8.h, v3.h ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) ; GFX1200-NEXT: v_cndmask_b16 v4.h, 0x7f, v4.h, s1 -; GFX1200-NEXT: v_or_b16 v8.h, v11.l, v10.l +; GFX1200-NEXT: v_add_nc_u16 v5.h, v6.h, v5.h ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo -; GFX1200-NEXT: v_or_b16 v3.l, v7.l, v3.l -; GFX1200-NEXT: v_and_b16 v7.l, v2.h, 1 -; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v8.l -; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v8.h -; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v13, 16, v1 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v6.l, vcc_lo -; GFX1200-NEXT: v_cndmask_b16 v2.l, 0, v6.h, s0 -; GFX1200-NEXT: v_or_b16 v5.l, v10.l, v7.l -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.h, v13.l -; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v12.l -; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v13.l -; GFX1200-NEXT: v_add_nc_u16 v2.l, v7.h, v2.l -; GFX1200-NEXT: v_and_b16 v5.l, v5.h, v5.l -; GFX1200-NEXT: v_sub_nc_u16 v5.h, 2, v6.h -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v9.h, vcc_lo -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v2.l -; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, v5.l -; GFX1200-NEXT: v_min_u16 v5.l, v5.h, 15 -; GFX1200-NEXT: v_lshrrev_b16 v5.h, 8, v1.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, vcc_lo -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.h -; GFX1200-NEXT: v_sub_nc_u16 v7.l, v5.l, 1 clamp -; GFX1200-NEXT: v_and_b16 v5.h, 0x80, v5.h -; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, 0, vcc_lo -; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v12.l, v12.l +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v9.l, vcc_lo +; GFX1200-NEXT: v_min_u16 v6.l, v6.l, 15 +; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.l, v2.l +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v3.h +; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v5.h +; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v2 +; GFX1200-NEXT: v_frexp_mant_f16_e32 v2.l, v1.l +; GFX1200-NEXT: v_sub_nc_u16 v2.h, v6.l, 1 clamp ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, s0 -; GFX1200-NEXT: v_and_b16 v8.h, 0x3ff, v6.l -; GFX1200-NEXT: v_lshlrev_b16 v9.h, v7.l, 1 -; GFX1200-NEXT: v_or_b16 v7.h, v5.h, v7.h -; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s0 -; GFX1200-NEXT: v_add_nc_u16 v8.l, v9.l, v8.l -; GFX1200-NEXT: v_or_b16 v9.l, 0x400, v8.h -; GFX1200-NEXT: v_add_nc_u16 v9.h, v9.h, -1 -; GFX1200-NEXT: v_or_b16 v2.l, v7.h, v2.l -; GFX1200-NEXT: v_and_b16 v7.h, v6.l, 63 -; GFX1200-NEXT: v_lshrrev_b16 v8.h, 7, v8.h -; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, 6 -; GFX1200-NEXT: v_and_b16 v9.h, v9.l, v9.h -; GFX1200-NEXT: v_lshrrev_b16 v10.l, v5.l, v9.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v7.h -; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l -; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v8.l -; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.l, v9.l -; GFX1200-NEXT: v_and_b16 v10.h, v10.l, 1 -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s1 -; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v9.h -; GFX1200-NEXT: v_and_b16 v9.h, v8.h, 1 -; GFX1200-NEXT: v_or_b16 v7.h, v5.h, v7.h -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.l, v0.l -; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v8.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s1 -; GFX1200-NEXT: v_or_b16 v9.h, v11.l, v9.h -; GFX1200-NEXT: v_or_b16 v2.h, v7.h, v2.h -; GFX1200-NEXT: v_sub_nc_u16 v7.h, 2, v9.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.l -; GFX1200-NEXT: v_or_b16 v10.h, v12.l, v10.h -; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v9.h -; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.h, v2.l, s0 -; GFX1200-NEXT: v_min_u16 v2.h, v7.h, 15 -; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.l -; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v10.h -; GFX1200-NEXT: v_add_nc_u16 v5.l, v8.h, v6.l +; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s0 +; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 1, s1 +; GFX1200-NEXT: v_lshlrev_b16 v7.h, 7, v9.l +; GFX1200-NEXT: v_and_b16 v8.h, 0x3ff, v2.l +; GFX1200-NEXT: v_lshlrev_b16 v9.l, v2.h, 1 +; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s0 +; GFX1200-NEXT: v_add_nc_u16 v7.l, v8.l, v7.l +; GFX1200-NEXT: v_or_b16 v6.h, v7.h, v6.h +; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v8.h +; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1 ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7f, v3.l, vcc_lo -; GFX1200-NEXT: v_sub_nc_u16 v7.h, v2.h, 1 clamp -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v5.h, s0 -; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, v7.l, s1 -; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v5.l -; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.l, v0.l -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h -; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, s2 -; GFX1200-NEXT: v_add_nc_u16 v6.l, v10.l, v6.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, s1 -; GFX1200-NEXT: v_and_b16 v5.h, 0x3ff, v7.l -; GFX1200-NEXT: v_lshlrev_b16 v8.h, v7.h, 1 -; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v1 -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v6.l -; GFX1200-NEXT: v_add_nc_u16 v1.h, v6.h, v8.l -; GFX1200-NEXT: v_or_b16 v6.h, 0x400, v5.h -; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.h, -1 -; GFX1200-NEXT: v_lshlrev_b16 v9.h, 7, v10.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 8, s0 -; GFX1200-NEXT: v_add_nc_u16 v1.h, v1.h, 6 -; GFX1200-NEXT: v_cndmask_b16 v6.l, v6.l, 0, s0 -; GFX1200-NEXT: v_and_b16 v1.l, v6.h, v8.l -; GFX1200-NEXT: v_lshrrev_b16 v10.l, v2.h, v6.h -; GFX1200-NEXT: v_or_b16 v8.l, v9.h, v8.h -; GFX1200-NEXT: v_lshlrev_b16 v8.h, 3, v1.h -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v10.h, v14.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v1.l -; GFX1200-NEXT: v_cndmask_b16 v1.l, v5.l, 0, s1 -; GFX1200-NEXT: v_or_b16 v6.l, v8.l, v6.l -; GFX1200-NEXT: v_or_b16 v5.l, v9.h, v8.h -; GFX1200-NEXT: v_and_b16 v8.h, v10.l, 1 -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0 -; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v1.h -; GFX1200-NEXT: v_lshrrev_b16 v6.h, v7.h, v6.h -; GFX1200-NEXT: v_or_b16 v1.l, v5.l, v1.l -; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v10.h -; GFX1200-NEXT: v_or_b16 v1.h, v11.l, v8.h -; GFX1200-NEXT: v_frexp_mant_f16_e32 v2.h, v14.l -; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v6.l, s0 -; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15 -; GFX1200-NEXT: v_and_b16 v6.l, v7.l, 63 -; GFX1200-NEXT: v_and_b16 v1.h, v6.h, v1.h -; GFX1200-NEXT: v_and_b16 v8.h, v2.h, 63 -; GFX1200-NEXT: v_and_b16 v8.l, v5.h, 1 -; GFX1200-NEXT: v_sub_nc_u16 v6.h, v5.l, 1 clamp -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l -; GFX1200-NEXT: v_and_b16 v6.l, 0x3ff, v2.h -; GFX1200-NEXT: v_lshrrev_b16 v7.l, 6, v7.l -; GFX1200-NEXT: v_lshrrev_b16 v2.h, 6, v2.h -; GFX1200-NEXT: v_lshlrev_b16 v7.h, v6.h, 1 -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0 -; GFX1200-NEXT: v_or_b16 v11.h, 0x400, v6.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.h -; GFX1200-NEXT: v_lshrrev_b16 v6.l, 7, v6.l -; GFX1200-NEXT: v_add_nc_u16 v7.h, v7.h, -1 -; GFX1200-NEXT: v_or_b16 v8.l, v11.l, v8.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v1.h, vcc_lo -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0 -; GFX1200-NEXT: v_and_b16 v8.h, v6.l, 1 -; GFX1200-NEXT: v_and_b16 v7.h, v11.h, v7.h -; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v8.l -; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v11.h -; GFX1200-NEXT: v_mov_b16_e32 v11.l, v12.l -; GFX1200-NEXT: v_lshrrev_b16 v12.l, v5.l, v11.h -; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.h -; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v7.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.l -; GFX1200-NEXT: v_or_b16 v7.h, v11.l, v8.h -; GFX1200-NEXT: v_and_b16 v8.l, v12.l, 1 -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v15, 0, 1, s0 -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h -; GFX1200-NEXT: v_add_nc_u16 v1.h, v10.l, v1.h -; GFX1200-NEXT: v_and_b16 v2.h, v2.h, v7.h -; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v13.l -; GFX1200-NEXT: v_mov_b16_e32 v11.l, v15.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo -; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v1.h -; GFX1200-NEXT: v_add_nc_u16 v2.h, v6.l, v2.h -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v9.h, s0 -; GFX1200-NEXT: v_or_b16 v7.l, v11.l, v8.l -; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo -; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s2 -; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v2.h -; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v13.l, v13.l -; GFX1200-NEXT: v_and_b16 v6.l, v6.h, v7.l -; GFX1200-NEXT: v_mov_b16_e32 v5.l, v8.l -; GFX1200-NEXT: v_lshrrev_b16 v6.h, 8, v0.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s3 -; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, s2 -; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, v6.l, s1 -; GFX1200-NEXT: v_add_nc_u16 v5.l, v9.l, v5.l -; GFX1200-NEXT: v_and_b16 v6.h, 0x80, v6.h -; GFX1200-NEXT: v_add_nc_u16 v7.l, v10.h, v7.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v0 -; GFX1200-NEXT: v_add_nc_u16 v6.l, v12.l, v6.l -; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, 6 -; GFX1200-NEXT: v_or_b16 v0.h, v6.h, v7.h ; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, 6 -; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v10.l -; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v6.l -; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v5.l -; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s3 +; GFX1200-NEXT: v_or_b16 v3.h, v6.h, v3.h +; GFX1200-NEXT: v_and_b16 v9.h, v2.l, 63 +; GFX1200-NEXT: v_and_b16 v6.h, v8.l, v9.l +; GFX1200-NEXT: v_lshrrev_b16 v10.l, v6.l, v8.l +; GFX1200-NEXT: v_lshrrev_b32_e32 v14, 16, v1 ; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v7.l -; GFX1200-NEXT: v_or_b16 v0.h, v0.h, v1.h -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 8, s1 -; GFX1200-NEXT: v_or_b16 v7.h, v6.h, v7.h -; GFX1200-NEXT: v_cndmask_b16 v6.l, v6.l, 0, s1 -; GFX1200-NEXT: v_or_b16 v9.l, v8.h, v9.l -; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v5.l -; GFX1200-NEXT: v_or_b16 v8.l, v8.h, v8.l -; GFX1200-NEXT: v_or_b16 v1.h, v7.h, v5.h -; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v7.l -; GFX1200-NEXT: v_or_b16 v2.h, v9.l, v2.h -; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, s0 -; GFX1200-NEXT: v_or_b16 v5.l, v8.l, v6.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v9.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.h +; GFX1200-NEXT: v_lshrrev_b16 v6.h, 7, v8.h +; GFX1200-NEXT: v_and_b16 v8.h, v10.l, 1 +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v10.h, v14.l +; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, s1 ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v0.h, v1.h, v0.h, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 1, vcc_lo +; GFX1200-NEXT: v_and_b16 v11.l, v6.h, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v11.h, 0, 1, s0 +; GFX1200-NEXT: v_or_b16 v9.l, v7.h, v9.l +; GFX1200-NEXT: v_lshrrev_b16 v2.h, v2.h, v8.l +; GFX1200-NEXT: v_or_b16 v8.h, v9.h, v8.h +; GFX1200-NEXT: v_sub_nc_u16 v9.h, 2, v10.h +; GFX1200-NEXT: v_or_b16 v8.l, v11.h, v11.l +; GFX1200-NEXT: v_lshrrev_b16 v2.l, 6, v2.l +; GFX1200-NEXT: v_or_b16 v5.h, v9.l, v5.h +; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v7.l +; GFX1200-NEXT: v_min_u16 v9.l, v9.h, 15 +; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.l, v14.l +; GFX1200-NEXT: v_and_b16 v2.l, v2.l, v8.l +; GFX1200-NEXT: v_and_b16 v2.h, v2.h, v8.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l +; GFX1200-NEXT: v_sub_nc_u16 v8.l, v9.l, 1 clamp +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v3.h, v5.h, v3.h, vcc_lo +; GFX1200-NEXT: v_add_nc_u16 v2.l, v6.h, v2.l +; GFX1200-NEXT: v_and_b16 v5.h, 0x3ff, v7.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.h, 0, v2.h, s0 +; GFX1200-NEXT: v_lshlrev_b16 v6.l, v8.l, 1 +; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v12.l +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.l +; GFX1200-NEXT: v_or_b16 v6.h, 0x400, v5.h +; GFX1200-NEXT: v_add_nc_u16 v2.h, v10.l, v2.h +; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, -1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v7.h, vcc_lo +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 1, s0 +; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v1.l +; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v2.h +; GFX1200-NEXT: v_and_b16 v6.l, v6.h, v6.l +; GFX1200-NEXT: v_and_b16 v11.h, v7.l, 63 +; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, v7.h +; GFX1200-NEXT: v_lshrrev_b16 v7.h, v9.l, v6.h +; GFX1200-NEXT: v_and_b16 v8.h, 0x80, v8.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v6.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, 8, s1 +; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, 6 +; GFX1200-NEXT: v_and_b16 v9.h, v7.h, 1 +; GFX1200-NEXT: v_lshrrev_b16 v6.h, v8.l, v6.h +; GFX1200-NEXT: v_cndmask_b16 v10.l, 0, 1, s2 +; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s1 +; GFX1200-NEXT: v_lshlrev_b16 v11.l, 3, v5.l +; GFX1200-NEXT: v_or_b16 v6.l, v8.h, v6.l +; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, 0, s0 +; GFX1200-NEXT: v_or_b16 v9.h, v10.l, v9.h +; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v11.h +; GFX1200-NEXT: v_or_b16 v8.l, v8.h, v11.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v9.l +; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v9.h +; GFX1200-NEXT: v_and_b16 v9.l, v5.h, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 1, s0 +; GFX1200-NEXT: v_or_b16 v2.h, v6.l, v2.h +; GFX1200-NEXT: v_or_b16 v2.l, v8.l, v2.l +; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v5.l +; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v6.h, s1 +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.h, v0.l +; GFX1200-NEXT: v_or_b16 v6.l, v9.h, v9.l +; GFX1200-NEXT: v_lshrrev_b16 v7.l, 6, v7.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v2.h, s0 +; GFX1200-NEXT: v_add_nc_u16 v2.h, v7.h, v5.l +; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v6.h +; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v12.l, v12.l +; GFX1200-NEXT: v_and_b16 v6.l, v7.l, v6.l +; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.l +; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v1 +; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v3.h, vcc_lo +; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v6.l +; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v0.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v8.h, s0 +; GFX1200-NEXT: v_sub_nc_u16 v7.l, v5.l, 1 clamp +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v2.h +; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v1.l, v1.l +; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v5.h +; GFX1200-NEXT: v_and_b16 v1.l, 0x3ff, v6.l +; GFX1200-NEXT: v_lshlrev_b16 v7.h, v7.l, 1 +; GFX1200-NEXT: v_lshrrev_b32_e32 v15, 16, v0 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, 8, vcc_lo +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 1, s1 +; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l +; GFX1200-NEXT: v_or_b16 v9.l, 0x400, v1.l +; GFX1200-NEXT: v_add_nc_u16 v7.h, v7.h, -1 +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.h, v15.l +; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, vcc_lo +; GFX1200-NEXT: v_add_nc_u16 v8.l, v10.h, v8.l +; GFX1200-NEXT: v_or_b16 v1.h, v8.h, v1.h +; GFX1200-NEXT: v_and_b16 v7.h, v9.l, v7.h +; GFX1200-NEXT: v_sub_nc_u16 v10.l, 2, v9.h +; GFX1200-NEXT: v_and_b16 v11.h, v6.l, 63 +; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, 6 +; GFX1200-NEXT: v_or_b16 v1.h, v1.h, v2.h +; GFX1200-NEXT: v_lshrrev_b16 v2.h, v5.l, v9.l +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.h +; GFX1200-NEXT: v_min_u16 v10.l, v10.l, 15 +; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v8.l +; GFX1200-NEXT: v_frexp_mant_f16_e32 v12.l, v15.l +; GFX1200-NEXT: v_and_b16 v10.h, v2.h, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v11.l, 0, 1, vcc_lo +; GFX1200-NEXT: v_sub_nc_u16 v12.h, v10.l, 1 clamp +; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, s1 +; GFX1200-NEXT: v_or_b16 v7.h, v8.h, v7.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v11.h +; GFX1200-NEXT: v_or_b16 v10.h, v11.l, v10.h +; GFX1200-NEXT: v_and_b16 v11.l, 0x3ff, v12.l +; GFX1200-NEXT: v_lshlrev_b16 v11.h, v12.h, 1 +; GFX1200-NEXT: v_lshrrev_b16 v1.l, 7, v1.l +; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.l, v9.l +; GFX1200-NEXT: v_or_b16 v5.h, v7.h, v5.h +; GFX1200-NEXT: v_or_b16 v13.h, 0x400, v11.l +; GFX1200-NEXT: v_add_nc_u16 v11.h, v11.h, -1 +; GFX1200-NEXT: v_and_b16 v7.h, v12.l, 63 +; GFX1200-NEXT: v_and_b16 v9.l, v1.l, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v13.l, 0, 1, vcc_lo +; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v10.h +; GFX1200-NEXT: v_and_b16 v10.h, v13.h, v11.h +; GFX1200-NEXT: v_lshrrev_b16 v11.l, 7, v11.l +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.h +; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, s0 +; GFX1200-NEXT: v_or_b16 v9.l, v13.l, v9.l +; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l +; GFX1200-NEXT: v_lshrrev_b16 v7.h, v10.l, v13.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v10.h +; GFX1200-NEXT: v_and_b16 v10.h, v11.l, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v11.h, 0, 1, vcc_lo +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l +; GFX1200-NEXT: v_and_b16 v5.l, v6.l, v9.l +; GFX1200-NEXT: v_and_b16 v6.l, v7.h, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v9.l, 0, 1, s0 +; GFX1200-NEXT: v_or_b16 v10.h, v11.h, v10.h +; GFX1200-NEXT: v_lshrrev_b16 v11.h, 6, v12.l +; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v8.l +; GFX1200-NEXT: v_add_nc_u16 v1.l, v1.l, v5.l +; GFX1200-NEXT: v_or_b16 v5.l, v9.l, v6.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, vcc_lo +; GFX1200-NEXT: v_and_b16 v6.l, v11.h, v10.h +; GFX1200-NEXT: v_lshrrev_b16 v8.l, v12.h, v13.h +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v1.h, v5.h, v1.h, s0 +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v1.l +; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, v7.l +; GFX1200-NEXT: v_add_nc_u16 v5.h, v11.l, v6.l +; GFX1200-NEXT: v_and_b16 v5.l, v8.l, v5.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v10.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, 1, vcc_lo +; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v2.h +; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v5.h +; GFX1200-NEXT: v_lshrrev_b16 v7.l, 8, v0.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, s0 +; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.h, v6.l +; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s1 +; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 1, s3 +; GFX1200-NEXT: v_and_b16 v7.l, 0x80, v7.l +; GFX1200-NEXT: v_add_nc_u16 v5.l, v7.h, v5.l +; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 6 +; GFX1200-NEXT: v_cmp_eq_f16_e64 s2, 0, v14.l +; GFX1200-NEXT: v_add_nc_u16 v7.h, v9.h, v8.l +; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v0 +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v5.l +; GFX1200-NEXT: v_or_b16 v0.h, v7.l, v6.h +; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s1 +; GFX1200-NEXT: v_add_nc_u16 v6.h, v7.h, 6 +; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v6.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 8, s0 +; GFX1200-NEXT: v_lshlrev_b16 v9.l, 7, v9.l +; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, 0, vcc_lo +; GFX1200-NEXT: v_lshlrev_b16 v9.h, 3, v6.h +; GFX1200-NEXT: v_or_b16 v7.h, v7.l, v7.h +; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, s0 +; GFX1200-NEXT: v_or_b16 v8.l, v9.l, v8.l +; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, s3 +; GFX1200-NEXT: v_or_b16 v9.h, v9.l, v9.h +; GFX1200-NEXT: v_or_b16 v0.h, v0.h, v2.h +; GFX1200-NEXT: v_or_b16 v1.l, v7.h, v1.l +; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v6.l +; GFX1200-NEXT: v_or_b16 v2.h, v8.l, v5.l +; GFX1200-NEXT: v_or_b16 v5.l, v9.h, v5.h +; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v6.h +; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v8.h, s2 +; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v14.l, v14.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v0.h, v1.l, v0.h, vcc_lo ; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v1.l, v5.l, v2.h, s0 +; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v15.l +; GFX1200-NEXT: v_cndmask_b16 v1.h, 0x7f, v1.h, s1 ; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v1.h, v2.h, v5.l, s1 -; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v14.l ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v6.h, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v7.l, vcc_lo ; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l -; GFX1200-NEXT: v_lshlrev_b16 v2.h, 8, v3.l -; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v1.l ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.h, v8.h, s1 -; GFX1200-NEXT: v_and_b16 v1.h, 0xff, v4.h -; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v14.l, v14.l -; GFX1200-NEXT: v_or_b16 v4.h, v4.l, v3.h -; GFX1200-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.l, v9.l, s0 +; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.h +; GFX1200-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX1200-NEXT: v_lshlrev_b16 v4.h, 8, v4.h +; GFX1200-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX1200-NEXT: v_lshlrev_b16 v2.h, 8, v3.h +; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v1.l +; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v15.l, v15.l +; GFX1200-NEXT: v_or_b16 v4.h, v4.l, v4.h ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX1200-NEXT: v_cndmask_b16 v0.h, 0x7f, v0.h, vcc_lo ; GFX1200-NEXT: v_or_b16 v4.l, v1.h, v2.h +; GFX1200-NEXT: v_lshlrev_b32_e32 v3, 16, v2 ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) ; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, s0 ; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l ; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX1200-NEXT: v_lshrrev_b64 v[6:7], 24, v[3:4] -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l ; GFX1200-NEXT: v_lshrrev_b32_e32 v5, 8, v4 +; GFX1200-NEXT: v_lshrrev_b64 v[6:7], 24, v[3:4] +; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l ; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1 ; GFX1200-NEXT: v_mov_b16_e32 v3.l, v6.l @@ -3206,439 +3168,431 @@ ; GFX1200-NEXT: s_wait_kmcnt 0x0 ; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.h, v2.l ; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.l, v2.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v12, 16, v2 -; GFX1200-NEXT: v_lshrrev_b32_e32 v13, 31, v2 +; GFX1200-NEXT: v_lshrrev_b32_e32 v13, 16, v2 ; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v2.l +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.l, v1.l ; GFX1200-NEXT: v_sub_nc_u16 v4.h, -5, v3.h -; GFX1200-NEXT: v_and_b16 v2.h, 0x3ff, v3.l -; GFX1200-NEXT: v_and_b16 v5.l, 0x7f, v3.l -; GFX1200-NEXT: v_lshrrev_b16 v3.l, 7, v3.l -; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v12.l +; GFX1200-NEXT: v_and_b16 v5.h, 0x3ff, v3.l +; GFX1200-NEXT: v_and_b16 v6.l, 0x7f, v3.l +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v7.l, v13.l +; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.h, v13.l ; GFX1200-NEXT: v_min_u16 v4.h, v4.h, 15 -; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v2.h -; GFX1200-NEXT: v_lshrrev_b16 v2.h, 8, v2.h -; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.l -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.h, v12.l -; GFX1200-NEXT: v_sub_nc_u16 v6.l, v4.h, 1 clamp -; GFX1200-NEXT: v_and_b16 v8.h, 0x3ff, v5.h -; GFX1200-NEXT: v_and_b16 v9.l, v2.h, 1 +; GFX1200-NEXT: v_or_b16 v8.h, 0x400, v5.h +; GFX1200-NEXT: v_lshrrev_b16 v5.h, 8, v5.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l +; GFX1200-NEXT: v_sub_nc_u16 v10.l, -5, v7.l +; GFX1200-NEXT: v_sub_nc_u16 v8.l, v4.h, 1 clamp +; GFX1200-NEXT: v_lshrrev_b16 v11.l, v4.h, v8.h +; GFX1200-NEXT: v_and_b16 v11.h, v5.h, 1 ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s1 -; GFX1200-NEXT: v_sub_nc_u16 v9.h, -5, v6.h -; GFX1200-NEXT: v_lshlrev_b16 v8.l, v6.l, 1 -; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v4.h -; GFX1200-NEXT: v_lshrrev_b16 v6.l, v6.l, v7.h -; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.l -; GFX1200-NEXT: v_lshrrev_b16 v10.l, v4.h, v7.h -; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1 -; GFX1200-NEXT: v_or_b16 v4.h, 0x400, v8.h -; GFX1200-NEXT: v_and_b16 v10.h, 0x7f, v5.h -; GFX1200-NEXT: v_and_b16 v3.l, v3.l, v9.l -; GFX1200-NEXT: v_and_b16 v9.l, v10.l, 1 -; GFX1200-NEXT: v_and_b16 v8.l, v7.h, v8.l -; GFX1200-NEXT: v_and_b16 v4.l, 0x80, v4.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s3, 0, v10.h -; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, v3.l -; GFX1200-NEXT: v_lshrrev_b16 v3.l, 8, v8.h -; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v8.l -; GFX1200-NEXT: v_min_u16 v8.l, v9.h, 15 -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s3 -; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h -; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l -; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s2 -; GFX1200-NEXT: v_sub_nc_u16 v7.h, v8.l, 1 clamp -; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 3, v2.h -; GFX1200-NEXT: v_cmp_ne_u16_e64 s3, 0, v8.l -; GFX1200-NEXT: v_cmp_class_f16_e64 s6, v2.l, 0x204 -; GFX1200-NEXT: v_or_b16 v8.h, v11.l, v9.l -; GFX1200-NEXT: v_lshrrev_b16 v9.l, v8.l, v4.h -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s2 -; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s2 -; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v2.l, v2.l -; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v8.h -; GFX1200-NEXT: v_lshlrev_b16 v8.h, v7.h, 1 -; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, v11.l -; GFX1200-NEXT: v_mov_b16_e32 v11.l, v14.l -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v2.l, v1.l -; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, v6.l, s1 -; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.h, -1 -; GFX1200-NEXT: v_and_b16 v8.h, v3.l, 1 -; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, 14 -; GFX1200-NEXT: v_and_b16 v9.h, v9.l, 1 -; GFX1200-NEXT: v_add_nc_u16 v6.l, v10.l, v6.l -; GFX1200-NEXT: v_and_b16 v8.l, v4.h, v8.l -; GFX1200-NEXT: v_lshrrev_b16 v4.h, v7.h, v4.h -; GFX1200-NEXT: v_lshlrev_b16 v7.h, 2, v3.h -; GFX1200-NEXT: v_cmp_gt_i16_e64 s2, 1, v3.h -; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v6.l -; GFX1200-NEXT: v_cmp_ne_u16_e64 s4, 0, v8.l -; GFX1200-NEXT: v_or_b16 v8.l, v11.l, v8.h -; GFX1200-NEXT: v_or_b16 v7.h, v4.l, v7.h -; GFX1200-NEXT: v_cmp_lt_i16_e64 s5, 30, v3.h -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 4, s1 -; GFX1200-NEXT: v_cndmask_b16 v6.l, v6.l, 0, s1 -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s4 -; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v2.h -; GFX1200-NEXT: v_or_b16 v2.h, v7.h, v2.h -; GFX1200-NEXT: v_or_b16 v8.h, v4.l, v8.h -; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v8.l -; GFX1200-NEXT: v_cmp_eq_u16_e64 s4, 30, v3.h -; GFX1200-NEXT: v_sub_nc_u16 v7.l, -5, v2.l -; GFX1200-NEXT: v_or_b16 v5.l, 0x7c, v4.l -; GFX1200-NEXT: v_or_b16 v6.l, v8.h, v6.l -; GFX1200-NEXT: v_or_b16 v9.h, v10.l, v9.h -; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v5.h -; GFX1200-NEXT: s_and_b32 s1, s4, s1 -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v6.l, s2 -; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX1200-NEXT: s_or_b32 s1, s5, s1 -; GFX1200-NEXT: v_and_b16 v3.h, v4.h, v9.h -; GFX1200-NEXT: v_min_u16 v4.h, v7.l, 15 -; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v5.l, s1 -; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v3.l -; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, v3.h, s3 -; GFX1200-NEXT: v_sub_nc_u16 v5.h, v4.h, 1 clamp -; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v4.h -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v4.l, vcc_lo -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s1 -; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.l, v1.l -; GFX1200-NEXT: v_add_nc_u16 v3.h, v9.l, v3.h -; GFX1200-NEXT: v_lshlrev_b16 v7.h, 7, v13.l -; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v5.l, s6 -; GFX1200-NEXT: v_mov_b16_e32 v6.l, v7.l -; GFX1200-NEXT: v_and_b16 v5.l, 0x3ff, v4.l -; GFX1200-NEXT: v_lshlrev_b16 v7.l, v5.h, 1 -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v3.h -; GFX1200-NEXT: v_and_b16 v8.h, 0x7f, v4.l -; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.h, v6.l -; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v5.l -; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, -1 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 4, vcc_lo -; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, vcc_lo -; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v5.l -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.h -; GFX1200-NEXT: v_and_b16 v7.l, v8.l, v7.l -; GFX1200-NEXT: v_or_b16 v6.h, v7.h, v6.h -; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 14 -; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v4.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l -; GFX1200-NEXT: v_and_b16 v7.l, v5.l, 1 -; GFX1200-NEXT: v_or_b16 v3.h, v6.h, v3.h -; GFX1200-NEXT: v_lshlrev_b16 v9.l, 2, v6.l -; GFX1200-NEXT: v_lshrrev_b16 v8.h, v4.h, v8.l -; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s1 -; GFX1200-NEXT: v_or_b16 v6.h, v10.l, v7.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo -; GFX1200-NEXT: v_or_b16 v9.l, v7.h, v9.l -; GFX1200-NEXT: v_and_b16 v9.h, v8.h, 1 -; GFX1200-NEXT: v_cndmask_b16 v2.h, 0x7e, v2.h, s0 -; GFX1200-NEXT: v_and_b16 v4.l, v4.l, v6.h -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v3.l -; GFX1200-NEXT: v_cmp_eq_u16_e64 s1, 30, v6.l -; GFX1200-NEXT: v_or_b16 v7.l, v9.l, v3.l -; GFX1200-NEXT: v_or_b16 v9.l, v11.l, v9.h -; GFX1200-NEXT: v_add_nc_u16 v4.l, v5.l, v4.l -; GFX1200-NEXT: v_lshrrev_b16 v5.h, v5.h, v8.l -; GFX1200-NEXT: s_and_b32 s1, s1, s0 -; GFX1200-NEXT: v_lshrrev_b32_e32 v13, 16, v1 -; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v6.l -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v4.l -; GFX1200-NEXT: v_and_b16 v3.l, v5.h, v9.l -; GFX1200-NEXT: v_or_b16 v4.h, 0x7c, v7.h -; GFX1200-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v3.h, v7.l, v3.h, vcc_lo -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0 -; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v3.l, s2 -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v13.l -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 30, v6.l -; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, s0 -; GFX1200-NEXT: v_add_nc_u16 v2.l, v2.l, v5.l -; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.h, v3.l -; GFX1200-NEXT: v_sub_nc_u16 v6.l, -5, v5.h -; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX1200-NEXT: s_or_b32 s1, vcc_lo, s1 -; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v1.l -; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.h, s1 -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v3.l -; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v12.l -; GFX1200-NEXT: v_add_nc_u16 v2.l, v2.l, 14 -; GFX1200-NEXT: v_min_u16 v6.l, v6.l, 15 -; GFX1200-NEXT: v_and_b16 v5.l, 0x80, v5.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 4, vcc_lo -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v7.h, s1 -; GFX1200-NEXT: v_lshlrev_b16 v7.l, 2, v2.l -; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.h, v13.l -; GFX1200-NEXT: v_sub_nc_u16 v8.l, v6.l, 1 clamp -; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, vcc_lo -; GFX1200-NEXT: v_or_b16 v6.h, v5.l, v6.h -; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l -; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v12.l, 0x204 -; GFX1200-NEXT: v_and_b16 v8.h, 0x3ff, v7.h +; GFX1200-NEXT: v_cndmask_b16 v12.l, 0, 1, s0 +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v4.h ; GFX1200-NEXT: v_lshlrev_b16 v9.l, v8.l, 1 -; GFX1200-NEXT: v_or_b16 v3.l, v6.h, v3.l -; GFX1200-NEXT: v_or_b16 v6.h, v7.l, v4.l -; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v2.l -; GFX1200-NEXT: v_or_b16 v7.l, 0x400, v8.h +; GFX1200-NEXT: v_lshrrev_b16 v3.l, 7, v3.l +; GFX1200-NEXT: v_and_b16 v10.h, 0x7f, v6.h +; GFX1200-NEXT: v_min_u16 v10.l, v10.l, 15 +; GFX1200-NEXT: v_or_b16 v11.h, v12.l, v11.h ; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1 +; GFX1200-NEXT: v_and_b16 v9.h, 0x3ff, v6.h +; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v10.h +; GFX1200-NEXT: v_and_b16 v10.h, v11.l, 1 +; GFX1200-NEXT: v_and_b16 v3.l, v3.l, v11.h +; GFX1200-NEXT: v_and_b16 v4.h, v8.h, v9.l +; GFX1200-NEXT: v_lshrrev_b16 v8.l, v8.l, v8.h +; GFX1200-NEXT: v_or_b16 v9.l, 0x400, v9.h +; GFX1200-NEXT: v_lshrrev_b16 v9.h, 8, v9.h +; GFX1200-NEXT: v_add_nc_u16 v3.l, v5.h, v3.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v4.h +; GFX1200-NEXT: v_sub_nc_u16 v4.h, v10.l, 1 clamp +; GFX1200-NEXT: v_lshrrev_b16 v5.h, v10.l, v9.l ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.h, s0 -; GFX1200-NEXT: v_and_b16 v4.h, 0x7f, v7.h -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v3.l, v6.h, v3.l, vcc_lo -; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v8.h -; GFX1200-NEXT: v_and_b16 v6.h, v7.l, v9.l -; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h -; GFX1200-NEXT: v_lshrrev_b16 v4.h, v6.l, v7.l -; GFX1200-NEXT: v_cmp_eq_u16_e64 s1, 30, v2.l -; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 30, v2.l -; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v12.l, v12.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.h -; GFX1200-NEXT: v_and_b16 v6.h, v8.h, 1 -; GFX1200-NEXT: v_lshrrev_b32_e32 v12, 16, v0 +; GFX1200-NEXT: v_cndmask_b16 v12.l, 0, 1, s2 +; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v10.l +; GFX1200-NEXT: v_cndmask_b16 v11.h, 0, 1, s1 +; GFX1200-NEXT: v_lshlrev_b16 v8.h, v4.h, 1 +; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v3.l +; GFX1200-NEXT: v_lshrrev_b16 v6.h, 7, v6.h +; GFX1200-NEXT: v_and_b16 v4.l, 0x80, v4.l +; GFX1200-NEXT: v_or_b16 v10.h, v11.h, v10.h +; GFX1200-NEXT: v_add_nc_u16 v8.h, v8.h, -1 +; GFX1200-NEXT: v_and_b16 v11.h, v9.h, 1 ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7e, v3.h, s0 -; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v1.l, 0x204 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v4.l -; GFX1200-NEXT: v_and_b16 v4.l, v4.h, 1 -; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v6.h -; GFX1200-NEXT: v_lshlrev_b16 v3.h, 8, v3.h -; GFX1200-NEXT: v_cmp_class_f16_e64 s5, v13.l, 0x204 -; GFX1200-NEXT: s_and_b32 s1, s1, vcc_lo -; GFX1200-NEXT: v_or_b16 v2.l, v10.l, v4.l -; GFX1200-NEXT: v_lshrrev_b16 v4.l, v8.l, v7.l -; GFX1200-NEXT: v_and_b16 v6.h, v7.h, v6.h -; GFX1200-NEXT: v_or_b16 v7.l, 0x7c, v5.l +; GFX1200-NEXT: v_cndmask_b16 v10.l, 0, 1, s1 +; GFX1200-NEXT: v_lshrrev_b16 v4.h, v4.h, v9.l +; GFX1200-NEXT: v_and_b16 v8.l, v8.l, v10.h +; GFX1200-NEXT: v_and_b16 v8.h, v9.l, v8.h +; GFX1200-NEXT: v_or_b16 v10.h, v12.l, v11.h +; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, v10.l +; GFX1200-NEXT: v_and_b16 v11.h, v5.h, 1 +; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, v8.l, s0 +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.h +; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v10.h +; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, 14 +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s1 +; GFX1200-NEXT: v_add_nc_u16 v8.l, v11.l, v8.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 1, s0 +; GFX1200-NEXT: v_add_nc_u16 v6.h, v9.h, v6.h +; GFX1200-NEXT: v_lshlrev_b16 v9.l, 2, v3.h +; GFX1200-NEXT: v_sub_nc_u16 v7.h, -5, v5.l +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v8.l +; GFX1200-NEXT: v_or_b16 v8.h, v8.h, v11.h +; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v6.h +; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 3, v3.l +; GFX1200-NEXT: v_cmp_eq_u16_e64 s4, 30, v3.h +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 4, s0 +; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v8.h +; GFX1200-NEXT: v_or_b16 v8.h, v4.l, v9.l +; GFX1200-NEXT: v_cndmask_b16 v8.l, v8.l, 0, s0 +; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v3.h +; GFX1200-NEXT: v_or_b16 v9.l, v4.l, v9.h +; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, s2 +; GFX1200-NEXT: v_or_b16 v3.l, v8.h, v3.l +; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 1, s1 +; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 30, v3.h +; GFX1200-NEXT: v_or_b16 v8.l, v9.l, v8.l +; GFX1200-NEXT: v_or_b16 v6.l, 0x7c, v4.l +; GFX1200-NEXT: v_add_nc_u16 v3.h, v5.h, v4.h +; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.l, v8.h +; GFX1200-NEXT: v_min_u16 v5.h, v7.h, 15 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v8.l, s0 +; GFX1200-NEXT: s_and_b32 s3, s4, s3 +; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l ; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX1200-NEXT: s_or_b32 s1, s2, s1 +; GFX1200-NEXT: s_or_b32 s2, s2, s3 +; GFX1200-NEXT: v_lshrrev_b32_e32 v14, 31, v2 +; GFX1200-NEXT: v_frexp_mant_f16_e32 v2.h, v1.l +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v3.h +; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v6.l, s2 +; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 14 +; GFX1200-NEXT: v_sub_nc_u16 v8.l, v5.h, 1 clamp +; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v14.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 4, s0 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v4.l, vcc_lo +; GFX1200-NEXT: v_lshlrev_b16 v4.l, 2, v4.h +; GFX1200-NEXT: v_and_b16 v8.h, 0x3ff, v2.h +; GFX1200-NEXT: v_lshlrev_b16 v9.l, v8.l, 1 +; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s0 +; GFX1200-NEXT: v_or_b16 v7.h, v7.l, v7.h +; GFX1200-NEXT: v_cndmask_b16 v6.h, v6.h, 0, s1 +; GFX1200-NEXT: v_or_b16 v4.l, v7.l, v4.l +; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v2.l, 0x204 +; GFX1200-NEXT: v_or_b16 v9.h, 0x400, v8.h +; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1 +; GFX1200-NEXT: v_or_b16 v3.h, v7.h, v3.h +; GFX1200-NEXT: v_or_b16 v4.l, v4.l, v6.h +; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v6.l, s0 +; GFX1200-NEXT: v_and_b16 v6.l, v9.h, v9.l +; GFX1200-NEXT: v_cmp_eq_u16_e64 s1, 30, v4.h +; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 30, v4.h +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v3.h, v4.l, v3.h, vcc_lo +; GFX1200-NEXT: v_lshrrev_b16 v4.l, v5.h, v9.h ; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.l -; GFX1200-NEXT: v_and_b16 v2.l, v4.l, v2.l -; GFX1200-NEXT: v_add_nc_u16 v4.l, v8.h, v6.h -; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, s1 -; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v1.l -; GFX1200-NEXT: v_or_b16 v2.h, v2.h, v3.h +; GFX1200-NEXT: v_and_b16 v4.h, 0x7f, v2.h +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v6.h +; GFX1200-NEXT: v_lshrrev_b16 v7.h, 8, v8.h +; GFX1200-NEXT: v_and_b16 v6.l, v4.l, 1 ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v2.l, 0, v2.l, vcc_lo -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v4.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v5.l, s1 -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.l, v0.l -; GFX1200-NEXT: v_add_nc_u16 v2.l, v4.h, v2.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo -; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.l, v1.l -; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, s0 -; GFX1200-NEXT: v_sub_nc_u16 v4.h, -5, v5.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v1 -; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v6.l -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v2.l -; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, vcc_lo -; GFX1200-NEXT: v_min_u16 v4.h, v4.h, 15 -; GFX1200-NEXT: v_lshlrev_b16 v6.h, 7, v7.l -; GFX1200-NEXT: v_add_nc_u16 v1.h, v5.h, 14 -; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v0.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v1.l, 0, 4, s0 -; GFX1200-NEXT: v_sub_nc_u16 v6.l, v4.h, 1 clamp -; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, 0, s0 -; GFX1200-NEXT: v_lshlrev_b16 v7.l, 2, v1.h -; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v5.h -; GFX1200-NEXT: v_or_b16 v1.l, v6.h, v1.l -; GFX1200-NEXT: v_lshlrev_b16 v8.l, v6.l, 1 -; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7e, v3.l, s1 -; GFX1200-NEXT: v_or_b16 v7.l, v6.h, v7.l -; GFX1200-NEXT: v_or_b16 v8.h, 0x400, v7.h -; GFX1200-NEXT: v_or_b16 v1.l, v1.l, v2.l -; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1 -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v4.l -; GFX1200-NEXT: v_or_b16 v2.l, v7.l, v4.l -; GFX1200-NEXT: v_cmp_eq_u16_e64 s0, 30, v1.h -; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v1.h -; GFX1200-NEXT: v_and_b16 v3.h, v8.h, v8.l -; GFX1200-NEXT: v_lshrrev_b16 v4.l, v4.h, v8.h -; GFX1200-NEXT: v_lshrrev_b16 v6.l, v6.l, v8.h -; GFX1200-NEXT: s_and_b32 s0, s0, vcc_lo -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 30, v1.h -; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v3.h -; GFX1200-NEXT: v_and_b16 v1.h, v4.l, 1 -; GFX1200-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, s1 -; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v2.l, v12.l -; GFX1200-NEXT: v_and_b16 v3.h, 0x7f, v5.h -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s2 -; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX1200-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h -; GFX1200-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.h -; GFX1200-NEXT: v_mov_b16_e32 v7.l, v9.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX1200-NEXT: v_or_b16 v1.h, v7.l, v1.h -; GFX1200-NEXT: v_lshrrev_b16 v7.l, 8, v7.h -; GFX1200-NEXT: v_sub_nc_u16 v7.h, -5, v2.l +; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 1, vcc_lo ; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h -; GFX1200-NEXT: v_and_b16 v1.h, v6.l, v1.h -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1200-NEXT: v_and_b16 v4.h, v7.l, 1 -; GFX1200-NEXT: v_min_u16 v3.h, v7.h, 15 -; GFX1200-NEXT: v_mov_b16_e32 v6.l, v8.l -; GFX1200-NEXT: v_frexp_mant_f16_e32 v8.l, v12.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v1.h, vcc_lo -; GFX1200-NEXT: v_or_b16 v7.h, 0x7c, v6.h -; GFX1200-NEXT: v_sub_nc_u16 v8.h, v3.h, 1 clamp -; GFX1200-NEXT: v_or_b16 v4.h, v6.l, v4.h -; GFX1200-NEXT: v_and_b16 v6.l, 0x3ff, v8.l -; GFX1200-NEXT: v_add_nc_u16 v1.h, v4.l, v1.h -; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v7.h, s0 -; GFX1200-NEXT: v_lshlrev_b16 v9.l, v8.h, 1 -; GFX1200-NEXT: v_and_b16 v4.l, v5.h, v4.h -; GFX1200-NEXT: v_or_b16 v4.h, 0x400, v6.l -; GFX1200-NEXT: v_lshrrev_b16 v6.l, 8, v6.l -; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v13.l -; GFX1200-NEXT: v_add_nc_u16 v5.h, v9.l, -1 -; GFX1200-NEXT: v_and_b16 v9.l, 0x7f, v8.l -; GFX1200-NEXT: v_lshrrev_b16 v8.l, 7, v8.l -; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.l, v4.l -; GFX1200-NEXT: v_lshrrev_b16 v7.l, v3.h, v4.h -; GFX1200-NEXT: v_and_b16 v5.h, v4.h, v5.h -; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v9.l -; GFX1200-NEXT: v_and_b16 v9.l, v6.l, 1 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v6.h, vcc_lo -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v1.h -; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v5.h -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s1 -; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v4.l -; GFX1200-NEXT: v_and_b16 v5.h, v7.l, 1 -; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s0 -; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s2 -; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.l -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s1 -; GFX1200-NEXT: v_lshrrev_b16 v4.h, v8.h, v4.h -; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v0.l -; GFX1200-NEXT: v_mov_b16_e32 v10.l, v14.l -; GFX1200-NEXT: v_and_b16 v6.h, v8.l, v9.l -; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 4, s0 -; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, v11.l -; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.h -; GFX1200-NEXT: v_or_b16 v5.h, v10.l, v5.h -; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, v6.h -; GFX1200-NEXT: v_and_b16 v3.h, 0x80, v8.h -; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, 14 -; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, s1 -; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v5.h -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v6.l -; GFX1200-NEXT: v_or_b16 v6.h, v3.h, v8.l -; GFX1200-NEXT: v_lshlrev_b16 v5.h, 2, v5.l -; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 3, v4.l -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, vcc_lo -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0 -; GFX1200-NEXT: v_or_b16 v1.h, v6.h, v1.h -; GFX1200-NEXT: v_or_b16 v5.h, v3.h, v5.h -; GFX1200-NEXT: v_cmp_eq_u16_e64 s3, 30, v5.l -; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.l, v4.h -; GFX1200-NEXT: v_mov_b16_e32 v8.l, v9.l -; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v5.l -; GFX1200-NEXT: v_or_b16 v5.h, v5.h, v4.l -; GFX1200-NEXT: v_cmp_lt_i16_e64 s4, 30, v5.l -; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v4.h -; GFX1200-NEXT: v_add_nc_u16 v2.l, v2.l, v8.l -; GFX1200-NEXT: v_lshrrev_b32_e32 v8, 31, v0 -; GFX1200-NEXT: v_cndmask_b16 v5.l, v6.l, 0, s0 -; GFX1200-NEXT: s_and_b32 s0, s3, s2 -; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, vcc_lo -; GFX1200-NEXT: v_add_nc_u16 v0.h, v2.l, 14 -; GFX1200-NEXT: v_cndmask_b16 v2.l, 0, 4, vcc_lo -; GFX1200-NEXT: v_lshlrev_b16 v4.l, 7, v8.l -; GFX1200-NEXT: v_cndmask_b16 v1.h, v5.h, v1.h, s1 -; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX1200-NEXT: s_or_b32 s3, s4, s0 -; GFX1200-NEXT: v_lshlrev_b16 v6.h, 2, v0.h -; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v0.h -; GFX1200-NEXT: v_or_b16 v2.l, v4.l, v2.l -; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v5.l -; GFX1200-NEXT: v_cmp_eq_u16_e64 s1, 30, v0.h -; GFX1200-NEXT: v_or_b16 v6.l, v4.l, v6.h -; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 30, v0.h -; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v4.h -; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v7.h, s5 +; GFX1200-NEXT: v_or_b16 v4.h, 0x7c, v7.l ; GFX1200-NEXT: s_and_b32 s0, s1, s0 -; GFX1200-NEXT: v_or_b16 v4.h, v6.l, v5.l -; GFX1200-NEXT: v_or_b16 v5.l, 0x7c, v3.h +; GFX1200-NEXT: v_lshrrev_b16 v2.h, 7, v2.h +; GFX1200-NEXT: v_or_b16 v6.l, v6.h, v6.l +; GFX1200-NEXT: v_lshrrev_b16 v6.h, v8.l, v9.h +; GFX1200-NEXT: v_and_b16 v8.l, v7.h, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 1, vcc_lo ; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) ; GFX1200-NEXT: s_or_b32 s0, s2, s0 -; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v13.l, v13.l -; GFX1200-NEXT: v_cmp_class_f16_e64 s2, v0.l, 0x204 +; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.l, v2.l +; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.l, v3.h, v4.h, s0 +; GFX1200-NEXT: v_and_b16 v3.h, v6.h, v6.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h +; GFX1200-NEXT: v_or_b16 v5.h, v8.h, v8.l +; GFX1200-NEXT: v_lshrrev_b32_e32 v11, 16, v1 +; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v13.l ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v0.h, v4.h, v2.l, vcc_lo -; GFX1200-NEXT: v_or_b16 v2.l, 0x7c, v4.l -; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v5.l, s3 +; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7e, v3.l, vcc_lo +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, v3.h, s0 +; GFX1200-NEXT: v_and_b16 v2.h, v2.h, v5.h +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v11.l +; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v7.l, s1 +; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v13.l, 0x204 +; GFX1200-NEXT: v_add_nc_u16 v3.h, v4.l, v3.h +; GFX1200-NEXT: v_add_nc_u16 v2.h, v7.h, v2.h +; GFX1200-NEXT: v_sub_nc_u16 v4.l, -5, v5.h +; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.l, v11.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v4.h, s0 +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v3.h +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v2.h +; GFX1200-NEXT: v_min_u16 v4.l, v4.l, 15 +; GFX1200-NEXT: v_lshrrev_b16 v4.h, 8, v1.l +; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v13.l, v13.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, 4, vcc_lo +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 1, s0 +; GFX1200-NEXT: v_sub_nc_u16 v7.h, v4.l, 1 clamp +; GFX1200-NEXT: v_and_b16 v4.h, 0x80, v4.h +; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s0 +; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, v6.h +; GFX1200-NEXT: v_and_b16 v6.h, 0x3ff, v7.l +; GFX1200-NEXT: v_lshlrev_b16 v8.l, v7.h, 1 +; GFX1200-NEXT: v_or_b16 v6.l, v4.h, v6.l +; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7e, v2.l, s1 +; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, 14 +; GFX1200-NEXT: v_or_b16 v8.h, 0x400, v6.h +; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1 +; GFX1200-NEXT: v_or_b16 v3.h, v6.l, v3.h +; GFX1200-NEXT: v_and_b16 v6.l, 0x7f, v7.l +; GFX1200-NEXT: v_lshrrev_b16 v6.h, 8, v6.h +; GFX1200-NEXT: v_lshlrev_b16 v9.l, 2, v5.l +; GFX1200-NEXT: v_and_b16 v8.l, v8.h, v8.l +; GFX1200-NEXT: v_lshrrev_b16 v7.l, 7, v7.l +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.l +; GFX1200-NEXT: v_cmp_eq_u16_e64 s1, 30, v5.l +; GFX1200-NEXT: v_or_b16 v6.l, v4.h, v9.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.l +; GFX1200-NEXT: v_and_b16 v8.l, v6.h, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 1, vcc_lo +; GFX1200-NEXT: v_lshrrev_b16 v9.l, v4.l, v8.h +; GFX1200-NEXT: v_or_b16 v6.l, v6.l, v2.h +; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v5.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v10.h, 0, 1, s0 +; GFX1200-NEXT: v_or_b16 v8.l, v9.h, v8.l +; GFX1200-NEXT: v_and_b16 v10.l, v9.l, 1 +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v2.h +; GFX1200-NEXT: v_lshrrev_b16 v7.h, v7.h, v8.h +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v3.h, v6.l, v3.h, vcc_lo +; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v8.l +; GFX1200-NEXT: v_or_b16 v2.h, v10.h, v10.l +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 30, v5.l +; GFX1200-NEXT: s_and_b32 s1, s1, s0 +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v4.l +; GFX1200-NEXT: v_add_nc_u16 v5.l, v6.h, v7.l +; GFX1200-NEXT: v_and_b16 v2.h, v7.h, v2.h +; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX1200-NEXT: s_or_b32 s1, vcc_lo, s1 +; GFX1200-NEXT: v_and_b16 v3.l, 0xff, v3.l +; GFX1200-NEXT: v_or_b16 v4.l, 0x7c, v4.h +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v5.l +; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v2.l +; GFX1200-NEXT: v_cndmask_b16 v2.h, 0, v2.h, s0 +; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.l +; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.l, s1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, 1, vcc_lo +; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v1 +; GFX1200-NEXT: v_add_nc_u16 v6.h, v9.l, v2.h +; GFX1200-NEXT: v_or_b16 v2.h, v3.l, v2.l +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v1.h, v0.l +; GFX1200-NEXT: v_add_nc_u16 v3.l, v5.h, v6.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.l, v3.h, v4.h, s0 +; GFX1200-NEXT: v_cmp_class_f16_e64 s1, v1.l, 0x204 +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v6.h +; GFX1200-NEXT: v_sub_nc_u16 v4.h, -5, v1.h +; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 14 +; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v4.l, s1 +; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, 4, s0 +; GFX1200-NEXT: v_lshlrev_b16 v4.l, 7, v7.l +; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.l, v1.l +; GFX1200-NEXT: v_lshlrev_b16 v1.l, 2, v3.l +; GFX1200-NEXT: v_min_u16 v4.h, v4.h, 15 +; GFX1200-NEXT: v_cndmask_b16 v5.h, v6.h, 0, s0 +; GFX1200-NEXT: v_or_b16 v3.h, v4.l, v3.h +; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v0.l +; GFX1200-NEXT: v_or_b16 v1.l, v4.l, v1.l +; GFX1200-NEXT: v_sub_nc_u16 v6.h, v4.h, 1 clamp +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v5.l +; GFX1200-NEXT: v_or_b16 v3.h, v3.h, v5.h +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7e, v2.l, s1 +; GFX1200-NEXT: v_or_b16 v1.l, v1.l, v5.l +; GFX1200-NEXT: v_and_b16 v5.l, 0x3ff, v6.l +; GFX1200-NEXT: v_lshlrev_b16 v5.h, v6.h, 1 +; GFX1200-NEXT: v_cmp_eq_u16_e64 s0, 30, v3.l +; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v3.l +; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 30, v3.l +; GFX1200-NEXT: v_or_b16 v3.l, 0x400, v5.l +; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, -1 +; GFX1200-NEXT: v_lshrrev_b32_e32 v12, 16, v0 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v3.h, s1 +; GFX1200-NEXT: s_and_b32 s0, s0, vcc_lo +; GFX1200-NEXT: v_and_b16 v8.l, 0x7f, v6.l +; GFX1200-NEXT: v_and_b16 v3.h, v3.l, v5.h +; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v7.l, v12.l +; GFX1200-NEXT: v_lshrrev_b16 v5.h, v4.h, v3.l +; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX1200-NEXT: s_or_b32 s1, s2, s0 +; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v5.l +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.h +; GFX1200-NEXT: v_sub_nc_u16 v8.h, -5, v7.l +; GFX1200-NEXT: v_and_b16 v3.h, v5.h, 1 +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.l +; GFX1200-NEXT: v_and_b16 v9.l, v5.l, 1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 1, vcc_lo +; GFX1200-NEXT: v_min_u16 v8.h, v8.h, 15 +; GFX1200-NEXT: v_lshrrev_b16 v3.l, v6.h, v3.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 1, s0 +; GFX1200-NEXT: v_lshrrev_b16 v6.l, 7, v6.l +; GFX1200-NEXT: v_or_b16 v3.h, v8.l, v3.h +; GFX1200-NEXT: v_frexp_mant_f16_e32 v8.l, v12.l +; GFX1200-NEXT: v_sub_nc_u16 v10.l, v8.h, 1 clamp +; GFX1200-NEXT: v_or_b16 v6.h, v9.h, v9.l +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h +; GFX1200-NEXT: v_and_b16 v3.l, v3.l, v3.h +; GFX1200-NEXT: v_and_b16 v9.l, 0x3ff, v8.l +; GFX1200-NEXT: v_lshlrev_b16 v9.h, v10.l, 1 +; GFX1200-NEXT: v_and_b16 v3.h, v6.l, v6.h +; GFX1200-NEXT: v_and_b16 v6.h, 0x7f, v8.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v3.l, vcc_lo +; GFX1200-NEXT: v_or_b16 v4.h, 0x400, v9.l +; GFX1200-NEXT: v_add_nc_u16 v6.l, v9.h, -1 +; GFX1200-NEXT: v_add_nc_u16 v3.h, v5.l, v3.h +; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.h +; GFX1200-NEXT: v_or_b16 v7.h, 0x7c, v4.l +; GFX1200-NEXT: v_lshrrev_b16 v6.h, v8.h, v4.h +; GFX1200-NEXT: v_and_b16 v5.l, v4.h, v6.l +; GFX1200-NEXT: v_lshrrev_b16 v6.l, 8, v9.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v9.l, 0, 1, vcc_lo +; GFX1200-NEXT: v_lshrrev_b16 v8.l, 7, v8.l +; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v7.h, s1 +; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v5.l +; GFX1200-NEXT: v_and_b16 v5.l, v6.l, 1 +; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v11.l +; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v3.h +; GFX1200-NEXT: v_add_nc_u16 v3.l, v5.h, v3.l +; GFX1200-NEXT: v_and_b16 v9.h, v6.h, 1 +; GFX1200-NEXT: v_or_b16 v5.l, v9.l, v5.l +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v10.h, 0, 1, s2 +; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, 1, s1 +; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s0 +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v3.l +; GFX1200-NEXT: v_and_b16 v5.l, v8.l, v5.l +; GFX1200-NEXT: v_or_b16 v4.l, v10.h, v9.h +; GFX1200-NEXT: v_lshrrev_b16 v4.h, v10.l, v4.h +; GFX1200-NEXT: v_lshrrev_b16 v8.l, 8, v0.l +; GFX1200-NEXT: v_add_nc_u16 v1.h, v1.h, v5.h +; GFX1200-NEXT: v_add_nc_u16 v5.l, v6.l, v5.l +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, 4, vcc_lo +; GFX1200-NEXT: v_and_b16 v4.l, v4.h, v4.l +; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.h +; GFX1200-NEXT: v_and_b16 v4.h, 0x80, v8.l +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, vcc_lo +; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v5.l +; GFX1200-NEXT: v_add_nc_u16 v1.h, v1.h, 14 +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v4.l, 0, v4.l, s0 +; GFX1200-NEXT: v_or_b16 v5.h, v4.h, v5.h +; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 1, vcc_lo +; GFX1200-NEXT: v_lshlrev_b16 v6.l, 2, v1.h +; GFX1200-NEXT: v_add_nc_u16 v4.l, v6.h, v4.l +; GFX1200-NEXT: v_or_b16 v3.l, v5.h, v3.l +; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 3, v3.h +; GFX1200-NEXT: v_add_nc_u16 v5.h, v7.l, v8.l +; GFX1200-NEXT: v_or_b16 v6.l, v4.h, v6.l +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v4.l +; GFX1200-NEXT: v_lshrrev_b32_e32 v8, 31, v0 +; GFX1200-NEXT: v_cmp_eq_u16_e64 s3, 30, v1.h +; GFX1200-NEXT: v_add_nc_u16 v0.h, v5.h, 14 +; GFX1200-NEXT: v_or_b16 v6.l, v6.l, v3.h +; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, 4, s0 +; GFX1200-NEXT: v_lshlrev_b16 v5.h, 7, v8.l +; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v1.h +; GFX1200-NEXT: v_lshlrev_b16 v6.h, 2, v0.h +; GFX1200-NEXT: v_cmp_lt_i16_e64 s4, 30, v1.h +; GFX1200-NEXT: v_cndmask_b16 v1.h, v4.l, 0, s0 +; GFX1200-NEXT: v_or_b16 v3.h, v5.h, v3.h +; GFX1200-NEXT: v_cndmask_b16 v4.l, v5.l, 0, vcc_lo +; GFX1200-NEXT: v_or_b16 v5.l, v5.h, v6.h +; GFX1200-NEXT: s_and_b32 s0, s3, s2 +; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v0.h +; GFX1200-NEXT: v_or_b16 v1.h, v3.h, v1.h +; GFX1200-NEXT: v_cndmask_b16 v3.l, v6.l, v3.l, s1 +; GFX1200-NEXT: v_or_b16 v3.h, v5.l, v4.l +; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX1200-NEXT: s_or_b32 s3, s4, s0 +; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v4.l +; GFX1200-NEXT: v_cmp_eq_u16_e64 s1, 30, v0.h +; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 30, v0.h +; GFX1200-NEXT: v_cmp_class_f16_e64 s5, v11.l, 0x204 +; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX1200-NEXT: v_cndmask_b16 v0.h, v3.h, v1.h, vcc_lo +; GFX1200-NEXT: v_or_b16 v1.h, 0x7c, v5.h +; GFX1200-NEXT: v_or_b16 v4.l, 0x7c, v4.h +; GFX1200-NEXT: s_and_b32 s0, s1, s0 +; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v7.h, s5 +; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX1200-NEXT: s_or_b32 s0, s2, s0 +; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v11.l, v11.l +; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v1.h, s0 +; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v12.l +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v4.l, s3 ; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) ; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7e, v1.l, s1 -; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v2.l, s0 -; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v12.l +; GFX1200-NEXT: v_cmp_class_f16_e64 s2, v0.l, 0x204 +; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v5.h, s0 +; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v12.l, 0x204 ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v3.h, vcc_lo +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v4.h, vcc_lo +; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l ; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v4.l, s0 -; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v12.l, 0x204 -; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v5.l, s2 -; GFX1200-NEXT: v_cvt_u32_u16_e32 v4, v2.h -; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX1200-NEXT: v_cndmask_b16 v0.l, v0.h, v2.l, s0 +; GFX1200-NEXT: v_cndmask_b16 v0.l, v0.h, v1.h, s0 ; GFX1200-NEXT: v_lshlrev_b16 v0.h, 8, v1.l +; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v4.l, s2 ; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v12.l, v12.l +; GFX1200-NEXT: v_cvt_u32_u16_e32 v4, v2.h +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v0.h ; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7e, v1.h, vcc_lo -; GFX1200-NEXT: v_or_b32_e32 v4, 0x7e7e0000, v4 -; GFX1200-NEXT: v_or_b16 v2.l, v3.l, v0.h +; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7e, v3.l, vcc_lo ; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0) ; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7e, v0.l, s0 -; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v1.l -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1200-NEXT: v_or_b32_e32 v4, 0x7e7e0000, v4 ; GFX1200-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v1.l +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l ; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1200-NEXT: v_lshrrev_b64 v[3:4], 24, v[3:4] -; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l -; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1 ; GFX1200-NEXT: v_mov_b16_e32 v4.l, v2.h +; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l +; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1 ; GFX1200-NEXT: s_setpc_b64 s[30:31] ; ; GFX1250-TRUE16-LABEL: to_bf8_v5f16:
diff --git a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll index b7f72c5..a6a0cdd 100644 --- a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll +++ b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
@@ -1909,11 +1909,8 @@ ; GFX11-SDAG-TRUE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0: ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4 @@ -1927,30 +1924,29 @@ ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3 +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2 ; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0 ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3 ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 ; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0 ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -2004,11 +2000,8 @@ ; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 +; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4 @@ -2027,29 +2020,28 @@ ; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2) ; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3 +; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2 ; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) ; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0 ; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 ; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 ; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
diff --git a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll index 785cb88..0b1b5a6 100644 --- a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
@@ -69,17 +69,16 @@ ; GFX11-SDAG-TRUE16-LABEL: v_test_fmed3_r_i_i_bf16_minimumnum_maximumnum: ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1 ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, 2.0, v1 ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 4.0, v1 ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4080, v0.l, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll index 7676c58..18178a9 100644 --- a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll +++ b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
@@ -1911,11 +1911,8 @@ ; GFX11-SDAG-TRUE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0: ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4 @@ -1929,30 +1926,29 @@ ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3 +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2 ; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0 ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3 ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 ; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0 ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -2006,11 +2002,8 @@ ; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 +; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4 @@ -2029,29 +2022,28 @@ ; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2) ; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3 +; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2 ; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) ; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0 ; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 ; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 ; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
diff --git a/llvm/test/CodeGen/AMDGPU/fract-match.ll b/llvm/test/CodeGen/AMDGPU/fract-match.ll index b40763b..babb4e7 100644 --- a/llvm/test/CodeGen/AMDGPU/fract-match.ll +++ b/llvm/test/CodeGen/AMDGPU/fract-match.ll
@@ -66,6 +66,7 @@ ; GFX6-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f32: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -81,6 +82,7 @@ ; GFX7-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f32: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -92,6 +94,7 @@ ; GFX8-NEXT: global_store_dword v[1:2], v3, off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_f32: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -102,6 +105,7 @@ ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc_lo ; GFX11-NEXT: global_store_b32 v[1:2], v4, off ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_f32: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -248,7 +252,6 @@ ; GFX6-IR-NEXT: [[COND6:%.*]] = select <3 x i1> [[CMPINF]], <3 x float> <float 0.000000e+00, float poison, float 0.000000e+00>, <3 x float> [[COND]] ; GFX6-IR-NEXT: store <3 x float> [[FLOOR]], ptr addrspace(1) [[IP]], align 4 ; GFX6-IR-NEXT: ret <3 x float> [[COND6]] -; ; IR-FRACT-LABEL: define <3 x float> @safe_math_fract_v3f32( ; IR-FRACT-SAME: <3 x float> [[X:%.*]], ptr addrspace(1) writeonly captures(none) [[IP:%.*]]) { ; IR-FRACT-NEXT: [[FLOOR:%.*]] = tail call <3 x float> @llvm.floor.v3f32(<3 x float> [[X]]) @@ -266,7 +269,6 @@ ; IR-FRACT-NEXT: [[COND6:%.*]] = select <3 x i1> [[CMPINF]], <3 x float> <float 0.000000e+00, float poison, float 0.000000e+00>, <3 x float> [[COND]] ; IR-FRACT-NEXT: store <3 x float> [[FLOOR]], ptr addrspace(1) [[IP]], align 4 ; IR-FRACT-NEXT: ret <3 x float> [[COND6]] -; %floor = tail call <3 x float> @llvm.floor.v3f32(<3 x float> %x) %sub = fsub <3 x float> %x, %floor %min = tail call <3 x float> @llvm.minnum.v3f32(<3 x float> %sub, <3 x float> <float 0x3FEFFFFFE0000000, float poison, float 0x3FEFFFFFE0000000>) @@ -316,6 +318,7 @@ ; GFX6-NEXT: buffer_store_dwordx2 v[4:5], v[2:3], s[4:7], 0 addr64 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_v2f32_const_splat_poison: ; GFX7: ; %bb.0: ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -339,6 +342,7 @@ ; GFX7-NEXT: buffer_store_dwordx2 v[4:5], v[2:3], s[4:7], 0 addr64 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_v2f32_const_splat_poison: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -358,6 +362,7 @@ ; GFX8-NEXT: global_store_dwordx2 v[2:3], v[4:5], off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_v2f32_const_splat_poison: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -376,6 +381,7 @@ ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_cndmask_b32_e64 v1, v7, 0, s0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_v2f32_const_splat_poison: ; GFX12: ; %bb.0: ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -458,6 +464,7 @@ ; GFX6-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f32_swap: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -473,6 +480,7 @@ ; GFX7-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f32_swap: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -484,6 +492,7 @@ ; GFX8-NEXT: global_store_dword v[1:2], v3, off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_f32_swap: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -494,6 +503,7 @@ ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc_lo ; GFX11-NEXT: global_store_b32 v[1:2], v4, off ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_f32_swap: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -557,6 +567,7 @@ ; GFX6-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f32_noinf_check: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -569,6 +580,7 @@ ; GFX7-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f32_noinf_check: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -577,6 +589,7 @@ ; GFX8-NEXT: global_store_dword v[1:2], v3, off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_f32_noinf_check: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -584,6 +597,7 @@ ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: global_store_b32 v[1:2], v3, off ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_f32_noinf_check: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -635,6 +649,7 @@ ; GFX6-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: no_nan_check_math_fract_f32: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -651,6 +666,7 @@ ; GFX7-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: no_nan_check_math_fract_f32: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -663,6 +679,7 @@ ; GFX8-NEXT: global_store_dword v[1:2], v3, off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: no_nan_check_math_fract_f32: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -674,6 +691,7 @@ ; GFX11-NEXT: v_min_f32_e32 v4, 0x3f7fffff, v4 ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v4, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: no_nan_check_math_fract_f32: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -723,21 +741,25 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: basic_fract_f32_nonans: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f32_e32 v0, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: basic_fract_f32_nonans: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f32_e32 v0, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: basic_fract_f32_nonans: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: basic_fract_f32_nonans: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -770,6 +792,7 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: basic_fract_f32_flags_minnum: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -777,6 +800,7 @@ ; GFX7-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX7-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: basic_fract_f32_flags_minnum: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -784,6 +808,7 @@ ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX8-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: basic_fract_f32_flags_minnum: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -792,6 +817,7 @@ ; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX11-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: basic_fract_f32_flags_minnum: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -833,21 +859,25 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: basic_fract_f32_flags_fsub: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f32_e32 v0, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: basic_fract_f32_flags_fsub: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f32_e32 v0, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: basic_fract_f32_flags_fsub: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: basic_fract_f32_flags_fsub: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -894,24 +924,28 @@ ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: v_min_f32_e32 v1, 0x3f7fffff, v1 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: basic_fract_v2f32_nonans: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f32_e32 v0, v0 ; GFX7-NEXT: v_fract_f32_e32 v1, v1 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: basic_fract_v2f32_nonans: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f32_e32 v0, v0 ; GFX8-NEXT: v_fract_f32_e32 v1, v1 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: basic_fract_v2f32_nonans: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: v_fract_f32_e32 v1, v1 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: basic_fract_v2f32_nonans: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -961,6 +995,7 @@ ; GFX6-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: basic_fract_f32_multi_use_fsub_nonans: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -974,6 +1009,7 @@ ; GFX7-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: basic_fract_f32_multi_use_fsub_nonans: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -983,6 +1019,7 @@ ; GFX8-NEXT: global_store_dword v[1:2], v3, off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: basic_fract_f32_multi_use_fsub_nonans: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -992,6 +1029,7 @@ ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: global_store_b32 v[1:2], v3, off ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: basic_fract_f32_multi_use_fsub_nonans: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1035,21 +1073,25 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: nnan_minnum_fract_f32: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f32_e32 v0, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: nnan_minnum_fract_f32: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f32_e32 v0, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: nnan_minnum_fract_f32: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: nnan_minnum_fract_f32: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1084,6 +1126,7 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: nnan_fsub_fract_f32: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1091,6 +1134,7 @@ ; GFX7-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX7-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: nnan_fsub_fract_f32: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1098,6 +1142,7 @@ ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX8-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: nnan_fsub_fract_f32: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1106,6 +1151,7 @@ ; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX11-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: nnan_fsub_fract_f32: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1141,6 +1187,7 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: nnan_floor_fract_f32: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1148,6 +1195,7 @@ ; GFX7-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX7-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: nnan_floor_fract_f32: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1155,6 +1203,7 @@ ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX8-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: nnan_floor_fract_f32: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1163,6 +1212,7 @@ ; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX11-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: nnan_floor_fract_f32: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1204,21 +1254,25 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: nnan_src_fract_f32: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f32_e32 v0, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: nnan_src_fract_f32: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f32_e32 v0, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: nnan_src_fract_f32: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: nnan_src_fract_f32: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1252,6 +1306,7 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7ffffe, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: not_fract_f32_wrong_const: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1259,6 +1314,7 @@ ; GFX7-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX7-NEXT: v_min_f32_e32 v0, 0x3f7ffffe, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: not_fract_f32_wrong_const: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1266,6 +1322,7 @@ ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX8-NEXT: v_min_f32_e32 v0, 0x3f7ffffe, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: not_fract_f32_wrong_const: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1274,6 +1331,7 @@ ; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX11-NEXT: v_min_f32_e32 v0, 0x3f7ffffe, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: not_fract_f32_wrong_const: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1310,6 +1368,7 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v1, v0 ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: not_fract_f32_swapped_fsub: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1317,6 +1376,7 @@ ; GFX7-NEXT: v_sub_f32_e32 v0, v1, v0 ; GFX7-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: not_fract_f32_swapped_fsub: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1324,6 +1384,7 @@ ; GFX8-NEXT: v_sub_f32_e32 v0, v1, v0 ; GFX8-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: not_fract_f32_swapped_fsub: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1332,6 +1393,7 @@ ; GFX11-NEXT: v_sub_f32_e32 v0, v1, v0 ; GFX11-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: not_fract_f32_swapped_fsub: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1368,6 +1430,7 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: not_fract_f32_not_floor: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1375,6 +1438,7 @@ ; GFX7-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX7-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: not_fract_f32_not_floor: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1382,6 +1446,7 @@ ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX8-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: not_fract_f32_not_floor: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1390,6 +1455,7 @@ ; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX11-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: not_fract_f32_not_floor: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1426,6 +1492,7 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: not_fract_f32_different_floor: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1433,6 +1500,7 @@ ; GFX7-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX7-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: not_fract_f32_different_floor: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1440,6 +1508,7 @@ ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX8-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: not_fract_f32_different_floor: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1448,6 +1517,7 @@ ; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX11-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: not_fract_f32_different_floor: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1484,6 +1554,7 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX6-NEXT: v_max_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: not_fract_f32_maxnum: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1491,6 +1562,7 @@ ; GFX7-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX7-NEXT: v_max_f32_e32 v0, 0x3f7fffff, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: not_fract_f32_maxnum: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1498,6 +1570,7 @@ ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX8-NEXT: v_max_f32_e32 v0, 0x3f7fffff, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: not_fract_f32_maxnum: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1506,6 +1579,7 @@ ; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX11-NEXT: v_max_f32_e32 v0, 0x3f7fffff, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: not_fract_f32_maxnum: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1540,6 +1614,7 @@ ; GCN: ; %bb.0: ; %entry ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: fcmp_uno_check_is_nan_f32: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1584,21 +1659,25 @@ ; GFX6-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: select_nan_fract_f32: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f32_e32 v0, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: select_nan_fract_f32: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f32_e32 v0, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: select_nan_fract_f32: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: select_nan_fract_f32: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1643,21 +1722,25 @@ ; GFX6-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: commuted_select_nan_fract_f32: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f32_e32 v0, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: commuted_select_nan_fract_f32: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f32_e32 v0, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: commuted_select_nan_fract_f32: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: commuted_select_nan_fract_f32: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1696,6 +1779,7 @@ ; GFX6-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: wrong_commuted_nan_select_f32: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1705,6 +1789,7 @@ ; GFX7-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: wrong_commuted_nan_select_f32: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1714,6 +1799,7 @@ ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: wrong_commuted_nan_select_f32: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1725,6 +1811,7 @@ ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: wrong_commuted_nan_select_f32: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1786,6 +1873,7 @@ ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fe000, v0 ; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: basic_fract_f16_nonan: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1799,21 +1887,25 @@ ; GFX7-NEXT: v_min_f32_e32 v0, 0x3f7fe000, v0 ; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: basic_fract_f16_nonan: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f16_e32 v0, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-TRUE16-LABEL: basic_fract_f16_nonan: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_fract_f16_e32 v0.l, v0.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-FAKE16-LABEL: basic_fract_f16_nonan: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: v_fract_f16_e32 v0, v0 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-TRUE16-LABEL: basic_fract_f16_nonan: ; GFX12-TRUE16: ; %bb.0: ; %entry ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1823,6 +1915,7 @@ ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_fract_f16_e32 v0.l, v0.l ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-FAKE16-LABEL: basic_fract_f16_nonan: ; GFX12-FAKE16: ; %bb.0: ; %entry ; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1892,6 +1985,7 @@ ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX6-NEXT: v_or_b32_e32 v0, v0, v1 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: basic_fract_v2f16_nonan: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1917,6 +2011,7 @@ ; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX7-NEXT: v_or_b32_e32 v0, v0, v1 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: basic_fract_v2f16_nonan: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1924,6 +2019,7 @@ ; GFX8-NEXT: v_fract_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX8-NEXT: v_pack_b32_f16 v0, v1, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-TRUE16-LABEL: basic_fract_v2f16_nonan: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1932,6 +2028,7 @@ ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_fract_f16_e32 v0.h, v1.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-FAKE16-LABEL: basic_fract_v2f16_nonan: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -1941,6 +2038,7 @@ ; GFX11-FAKE16-NEXT: v_fract_f16_e32 v1, v1 ; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-TRUE16-LABEL: basic_fract_v2f16_nonan: ; GFX12-TRUE16: ; %bb.0: ; %entry ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -1953,6 +2051,7 @@ ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX12-TRUE16-NEXT: v_fract_f16_e32 v0.h, v1.l ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-FAKE16-LABEL: basic_fract_v2f16_nonan: ; GFX12-FAKE16: ; %bb.0: ; %entry ; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -2004,21 +2103,25 @@ ; GFX6-NEXT: v_add_f64 v[0:1], v[0:1], -v[2:3] ; GFX6-NEXT: v_min_f64 v[0:1], v[0:1], s[4:5] ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: basic_fract_f64_nanans: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f64_e32 v[0:1], v[0:1] ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: basic_fract_f64_nanans: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f64_e32 v[0:1], v[0:1] ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: basic_fract_f64_nanans: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f64_e32 v[0:1], v[0:1] ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: basic_fract_f64_nanans: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -2087,6 +2190,7 @@ ; GFX6-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f16_noinf_check: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2108,6 +2212,7 @@ ; GFX7-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f16_noinf_check: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2116,6 +2221,7 @@ ; GFX8-NEXT: global_store_short v[1:2], v3, off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-TRUE16-LABEL: safe_math_fract_f16_noinf_check: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2123,6 +2229,7 @@ ; GFX11-TRUE16-NEXT: v_fract_f16_e32 v0.l, v0.l ; GFX11-TRUE16-NEXT: global_store_d16_hi_b16 v[1:2], v0, off ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-FAKE16-LABEL: safe_math_fract_f16_noinf_check: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2130,6 +2237,7 @@ ; GFX11-FAKE16-NEXT: v_fract_f16_e32 v0, v0 ; GFX11-FAKE16-NEXT: global_store_b16 v[1:2], v3, off ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-TRUE16-LABEL: safe_math_fract_f16_noinf_check: ; GFX12-TRUE16: ; %bb.0: ; %entry ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -2141,6 +2249,7 @@ ; GFX12-TRUE16-NEXT: v_fract_f16_e32 v0.l, v0.l ; GFX12-TRUE16-NEXT: global_store_d16_hi_b16 v[1:2], v0, off ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-FAKE16-LABEL: safe_math_fract_f16_noinf_check: ; GFX12-FAKE16: ; %bb.0: ; %entry ; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -2207,6 +2316,7 @@ ; GFX6-NEXT: buffer_store_dwordx2 v[4:5], v[2:3], s[4:7], 0 addr64 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f64_noinf_check: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2219,6 +2329,7 @@ ; GFX7-NEXT: buffer_store_dwordx2 v[4:5], v[2:3], s[4:7], 0 addr64 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f64_noinf_check: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2227,6 +2338,7 @@ ; GFX8-NEXT: global_store_dwordx2 v[2:3], v[4:5], off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_f64_noinf_check: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2234,6 +2346,7 @@ ; GFX11-NEXT: v_fract_f64_e32 v[0:1], v[0:1] ; GFX11-NEXT: global_store_b64 v[2:3], v[4:5], off ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_f64_noinf_check: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -2281,21 +2394,25 @@ ; GFX6-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: select_nan_fract_f32_flags_select: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f32_e32 v0, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: select_nan_fract_f32_flags_select: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f32_e32 v0, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: select_nan_fract_f32_flags_select: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: select_nan_fract_f32_flags_select: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -2340,21 +2457,25 @@ ; GFX6-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: select_nan_fract_f32_flags_minnum: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f32_e32 v0, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: select_nan_fract_f32_flags_minnum: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f32_e32 v0, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: select_nan_fract_f32_flags_minnum: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: select_nan_fract_f32_flags_minnum: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -2429,6 +2550,7 @@ ; GFX6-NEXT: buffer_store_dwordx2 v[4:5], v[2:3], s[4:7], 0 addr64 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_v2f32: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2448,6 +2570,7 @@ ; GFX7-NEXT: buffer_store_dwordx2 v[4:5], v[2:3], s[4:7], 0 addr64 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_v2f32: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2463,6 +2586,7 @@ ; GFX8-NEXT: global_store_dwordx2 v[2:3], v[4:5], off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_v2f32: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2477,6 +2601,7 @@ ; GFX11-NEXT: global_store_b64 v[2:3], v[4:5], off ; GFX11-NEXT: v_cndmask_b32_e64 v1, v7, 0, s0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_v2f32: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -2565,6 +2690,7 @@ ; GFX6-NEXT: buffer_store_dwordx2 v[4:5], v[2:3], s[4:7], 0 addr64 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f64: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2582,6 +2708,7 @@ ; GFX7-NEXT: buffer_store_dwordx2 v[6:7], v[2:3], s[4:7], 0 addr64 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f64: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2595,6 +2722,7 @@ ; GFX8-NEXT: global_store_dwordx2 v[2:3], v[6:7], off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_f64: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2605,6 +2733,7 @@ ; GFX11-NEXT: v_dual_cndmask_b32 v0, 0, v4 :: v_dual_cndmask_b32 v1, 0, v5 ; GFX11-NEXT: global_store_b64 v[2:3], v[6:7], off ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_f64: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -2698,6 +2827,7 @@ ; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f16: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2723,6 +2853,7 @@ ; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f16: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2734,6 +2865,7 @@ ; GFX8-NEXT: global_store_short v[1:2], v3, off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-TRUE16-LABEL: safe_math_fract_f16: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2744,6 +2876,7 @@ ; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, v0.h, s0 ; GFX11-TRUE16-NEXT: global_store_b16 v[1:2], v3, off ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-FAKE16-LABEL: safe_math_fract_f16: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2754,6 +2887,7 @@ ; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc_lo ; GFX11-FAKE16-NEXT: global_store_b16 v[1:2], v4, off ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-TRUE16-LABEL: safe_math_fract_f16: ; GFX12-TRUE16: ; %bb.0: ; %entry ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -2769,6 +2903,7 @@ ; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, v0.h, s0 ; GFX12-TRUE16-NEXT: global_store_b16 v[1:2], v3, off ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-FAKE16-LABEL: safe_math_fract_f16: ; GFX12-FAKE16: ; %bb.0: ; %entry ; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -2887,6 +3022,7 @@ ; GFX6-NEXT: v_or_b32_e32 v0, v4, v0 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_v2f16: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2932,6 +3068,7 @@ ; GFX7-NEXT: v_or_b32_e32 v0, v4, v0 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_v2f16: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2949,6 +3086,7 @@ ; GFX8-NEXT: global_store_dword v[1:2], v3, off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-TRUE16-LABEL: safe_math_fract_v2f16: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2964,6 +3102,7 @@ ; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, 0, s0 ; GFX11-TRUE16-NEXT: global_store_b32 v[1:2], v5, off ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-FAKE16-LABEL: safe_math_fract_v2f16: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -2982,6 +3121,7 @@ ; GFX11-FAKE16-NEXT: global_store_b32 v[1:2], v4, off ; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v3 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-TRUE16-LABEL: safe_math_fract_v2f16: ; GFX12-TRUE16: ; %bb.0: ; %entry ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -3002,6 +3142,7 @@ ; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, 0, s0 ; GFX12-TRUE16-NEXT: global_store_b32 v[1:2], v5, off ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-FAKE16-LABEL: safe_math_fract_v2f16: ; GFX12-FAKE16: ; %bb.0: ; %entry ; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -3113,6 +3254,7 @@ ; GFX6-NEXT: buffer_store_dwordx4 v[6:9], v[4:5], s[4:7], 0 addr64 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_v2f64: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3134,6 +3276,7 @@ ; GFX7-NEXT: buffer_store_dwordx4 v[6:9], v[4:5], s[8:11], 0 addr64 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_v2f64: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3151,6 +3294,7 @@ ; GFX8-NEXT: global_store_dwordx4 v[4:5], v[6:9], off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_v2f64: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3166,6 +3310,7 @@ ; GFX11-NEXT: v_cndmask_b32_e64 v3, v13, 0, s1 ; GFX11-NEXT: global_store_b128 v[4:5], v[6:9], off ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_v2f64: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -3246,6 +3391,7 @@ ; GFX6-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f32_minimum: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3261,6 +3407,7 @@ ; GFX7-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f32_minimum: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3272,6 +3419,7 @@ ; GFX8-NEXT: global_store_dword v[1:2], v3, off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_f32_minimum: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3282,6 +3430,7 @@ ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc_lo ; GFX11-NEXT: global_store_b32 v[1:2], v4, off ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_f32_minimum: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -3357,6 +3506,7 @@ ; GFX6-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f32_minimum_swap: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3372,6 +3522,7 @@ ; GFX7-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f32_minimum_swap: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3383,6 +3534,7 @@ ; GFX8-NEXT: global_store_dword v[1:2], v3, off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_f32_minimum_swap: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3393,6 +3545,7 @@ ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc_lo ; GFX11-NEXT: global_store_b32 v[1:2], v4, off ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_f32_minimum_swap: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -3465,6 +3618,7 @@ ; GFX6-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f32_minimumnum: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3480,6 +3634,7 @@ ; GFX7-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f32_minimumnum: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3491,6 +3646,7 @@ ; GFX8-NEXT: global_store_dword v[1:2], v3, off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_f32_minimumnum: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3501,6 +3657,7 @@ ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc_lo ; GFX11-NEXT: global_store_b32 v[1:2], v4, off ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_f32_minimumnum: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -3573,6 +3730,7 @@ ; GFX6-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f32_minimumnum_swap: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3588,6 +3746,7 @@ ; GFX7-NEXT: buffer_store_dword v3, v[1:2], s[4:7], 0 addr64 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f32_minimumnum_swap: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3599,6 +3758,7 @@ ; GFX8-NEXT: global_store_dword v[1:2], v3, off ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_f32_minimumnum_swap: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3609,6 +3769,7 @@ ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc_lo ; GFX11-NEXT: global_store_b32 v[1:2], v4, off ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_f32_minimumnum_swap: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -3659,21 +3820,25 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: basic_fract_f32_nonans_minimumnum: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f32_e32 v0, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: basic_fract_f32_nonans_minimumnum: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f32_e32 v0, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: basic_fract_f32_nonans_minimumnum: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: basic_fract_f32_nonans_minimumnum: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -3715,21 +3880,25 @@ ; GFX6-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: basic_fract_f32_nonans_minimum: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f32_e32 v0, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: basic_fract_f32_nonans_minimum: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f32_e32 v0, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: basic_fract_f32_nonans_minimum: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: basic_fract_f32_nonans_minimum: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -3768,21 +3937,25 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: nnan_minimum_fract_f32: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f32_e32 v0, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: nnan_minimum_fract_f32: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f32_e32 v0, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: nnan_minimum_fract_f32: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: nnan_minimum_fract_f32: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -3821,21 +3994,25 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: nnan_minimumnum_fract_f32: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f32_e32 v0, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: nnan_minimumnum_fract_f32: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f32_e32 v0, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: nnan_minimumnum_fract_f32: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: nnan_minimumnum_fract_f32: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -3868,6 +4045,7 @@ ; GFX6-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX6-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: basic_fract_f32_flags_minimumnum: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3875,6 +4053,7 @@ ; GFX7-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX7-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: basic_fract_f32_flags_minimumnum: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3882,6 +4061,7 @@ ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX8-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: basic_fract_f32_flags_minimumnum: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -3890,6 +4070,7 @@ ; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX11-NEXT: v_min_f32_e32 v0, 0x3f7fffff, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: basic_fract_f32_flags_minimumnum: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -3934,21 +4115,25 @@ ; GFX6-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: basic_fract_f32_flags_minimum: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f32_e32 v0, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: basic_fract_f32_flags_minimum: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f32_e32 v0, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: basic_fract_f32_flags_minimum: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: basic_fract_f32_flags_minimum: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -4044,7 +4229,6 @@ ; GFX6-IR-NEXT: [[IS_INF:%.*]] = fcmp oeq double [[X_ABS]], +inf ; GFX6-IR-NEXT: [[RESULT:%.*]] = select i1 [[IS_INF]], double 0.000000e+00, double [[MIN]] ; GFX6-IR-NEXT: ret double [[RESULT]] -; ; IR-FRACT-LABEL: define double @fract_match_f64_assume_not_nan( ; IR-FRACT-SAME: double [[X:%.*]]) #[[ATTR0:[0-9]+]] { ; IR-FRACT-NEXT: [[ENTRY:.*:]] @@ -4055,7 +4239,6 @@ ; IR-FRACT-NEXT: [[IS_INF:%.*]] = fcmp oeq double [[X_ABS]], +inf ; IR-FRACT-NEXT: [[RESULT:%.*]] = select i1 [[IS_INF]], double 0.000000e+00, double [[MIN]] ; IR-FRACT-NEXT: ret double [[RESULT]] -; entry: %is.ord = fcmp ord double %x, 0.000000e+00 tail call void @llvm.assume(i1 %is.ord) @@ -4104,6 +4287,7 @@ ; GFX6-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f32_swapped_edge_case: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4112,6 +4296,7 @@ ; GFX7-NEXT: v_cmp_neq_f32_e64 vcc, |v0|, s4 ; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f32_swapped_edge_case: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4120,6 +4305,7 @@ ; GFX8-NEXT: v_cmp_neq_f32_e64 vcc, |v0|, s4 ; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_f32_swapped_edge_case: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4128,6 +4314,7 @@ ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_f32_swapped_edge_case: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -4235,7 +4422,6 @@ ; GFX6-IR-NEXT: [[NOT_NAN:%.*]] = fcmp ord float [[X]], 0.000000e+00 ; GFX6-IR-NEXT: [[COND8:%.*]] = select i1 [[NOT_NAN]], float [[COND]], float [[X]] ; GFX6-IR-NEXT: ret float [[COND8]] -; ; IR-FRACT-LABEL: define float @safe_math_fract_f32_swapped_edge_case_multi_use_inner_select( ; IR-FRACT-SAME: float [[X:%.*]], ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] { ; IR-FRACT-NEXT: [[COND:%.*]] = call float @llvm.amdgcn.fract.f32(float [[X]]) @@ -4244,7 +4430,6 @@ ; IR-FRACT-NEXT: [[COND8:%.*]] = select i1 [[NOT_INF]], float [[COND]], float 0.000000e+00 ; IR-FRACT-NEXT: store float [[COND8]], ptr addrspace(1) [[PTR]], align 4 ; IR-FRACT-NEXT: ret float [[COND8]] -; %floor = call float @llvm.floor.f32(float %x) %sub = fsub float %x, %floor %min = call float @llvm.minnum.f32(float %sub, float 0x3FEFFFFFE0000000) @@ -4344,7 +4529,6 @@ ; GFX6-IR-NEXT: [[NOT_NAN:%.*]] = fcmp ord float [[X]], 0.000000e+00 ; GFX6-IR-NEXT: [[COND8:%.*]] = select i1 [[NOT_NAN]], float [[COND]], float [[X]] ; GFX6-IR-NEXT: ret float [[COND8]] -; ; IR-FRACT-LABEL: define float @safe_math_fract_f32_swapped_edge_case_multi_use_inner_select_fcmp( ; IR-FRACT-SAME: float [[X:%.*]], ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] { ; IR-FRACT-NEXT: [[COND:%.*]] = call float @llvm.amdgcn.fract.f32(float [[X]]) @@ -4353,7 +4537,6 @@ ; IR-FRACT-NEXT: store i1 [[NOT_INF]], ptr addrspace(1) [[PTR]], align 1 ; IR-FRACT-NEXT: [[COND8:%.*]] = select i1 [[NOT_INF]], float [[COND]], float 0.000000e+00 ; IR-FRACT-NEXT: ret float [[COND8]] -; %floor = call float @llvm.floor.f32(float %x) %sub = fsub float %x, %floor %min = call float @llvm.minnum.f32(float %sub, float 0x3FEFFFFFE0000000) @@ -4453,7 +4636,6 @@ ; GFX6-IR-NEXT: [[NOT_NAN:%.*]] = fcmp ord float [[X]], 0.000000e+00 ; GFX6-IR-NEXT: [[COND8:%.*]] = select i1 [[NOT_NAN]], float [[COND]], float [[X]] ; GFX6-IR-NEXT: ret float [[COND8]] -; ; IR-FRACT-LABEL: define float @safe_math_fract_f32_swapped_edge_case_multi_use_fabs( ; IR-FRACT-SAME: float [[X:%.*]], ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] { ; IR-FRACT-NEXT: [[COND:%.*]] = call float @llvm.amdgcn.fract.f32(float [[X]]) @@ -4462,7 +4644,6 @@ ; IR-FRACT-NEXT: [[NOT_INF:%.*]] = fcmp une float [[X_FABS]], +inf ; IR-FRACT-NEXT: [[COND8:%.*]] = select i1 [[NOT_INF]], float [[COND]], float 0.000000e+00 ; IR-FRACT-NEXT: ret float [[COND8]] -; %floor = call float @llvm.floor.f32(float %x) #3 %sub = fsub float %x, %floor %min = call float @llvm.minnum.f32(float %sub, float 0x3FEFFFFFE0000000) @@ -4501,6 +4682,7 @@ ; GFX6-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f32_swapped_edge_case_wrong_compared: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4513,6 +4695,7 @@ ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f32_swapped_edge_case_wrong_compared: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4525,6 +4708,7 @@ ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_f32_swapped_edge_case_wrong_compared: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4538,6 +4722,7 @@ ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v0 ; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_f32_swapped_edge_case_wrong_compared: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -4595,6 +4780,7 @@ ; GFX6-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f32_swapped_edge_case_commute_inf_check: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4607,6 +4793,7 @@ ; GFX7-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f32_swapped_edge_case_commute_inf_check: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4619,6 +4806,7 @@ ; GFX8-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_f32_swapped_edge_case_commute_inf_check: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4632,6 +4820,7 @@ ; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v0 ; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_f32_swapped_edge_case_commute_inf_check: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -4689,6 +4878,7 @@ ; GFX6-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f32_swapped_edge_case_commute_nan_check: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4701,6 +4891,7 @@ ; GFX7-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 ; GFX7-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f32_swapped_edge_case_commute_nan_check: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4713,6 +4904,7 @@ ; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_f32_swapped_edge_case_commute_nan_check: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4726,6 +4918,7 @@ ; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 ; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_f32_swapped_edge_case_commute_nan_check: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -4781,21 +4974,25 @@ ; GFX6-NEXT: v_cmp_o_f32_e32 vcc, v0, v0 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: safe_math_fract_f32_swapped_edge_case_cmp_neg_inf: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-NEXT: v_fract_f32_e32 v0, v0 ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: safe_math_fract_f32_swapped_edge_case_cmp_neg_inf: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_fract_f32_e32 v0, v0 ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: safe_math_fract_f32_swapped_edge_case_cmp_neg_inf: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_fract_f32_e32 v0, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: safe_math_fract_f32_swapped_edge_case_cmp_neg_inf: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -4839,6 +5036,7 @@ ; GFX6-NEXT: v_cmp_neq_f32_e64 vcc, |v0|, s4 ; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: basic_fract_f32_with_inf_check: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4849,6 +5047,7 @@ ; GFX7-NEXT: v_cmp_neq_f32_e64 vcc, |v0|, s4 ; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: basic_fract_f32_with_inf_check: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4859,6 +5058,7 @@ ; GFX8-NEXT: v_cmp_neq_f32_e64 vcc, |v0|, s4 ; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: basic_fract_f32_with_inf_check: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4870,6 +5070,7 @@ ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: basic_fract_f32_with_inf_check: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -4927,6 +5128,7 @@ ; GFX6-NEXT: v_cmp_lg_f32_e64 vcc, |v0|, s4 ; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc ; GFX6-NEXT: s_setpc_b64 s[30:31] +; ; GFX7-LABEL: basic_fract_f32_nonans_with_inf_check: ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4935,6 +5137,7 @@ ; GFX7-NEXT: v_cmp_lg_f32_e64 vcc, |v0|, s4 ; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc ; GFX7-NEXT: s_setpc_b64 s[30:31] +; ; GFX8-LABEL: basic_fract_f32_nonans_with_inf_check: ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4943,6 +5146,7 @@ ; GFX8-NEXT: v_cmp_lg_f32_e64 vcc, |v0|, s4 ; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc ; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX11-LABEL: basic_fract_f32_nonans_with_inf_check: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) @@ -4951,6 +5155,7 @@ ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] +; ; GFX12-LABEL: basic_fract_f32_nonans_with_inf_check: ; GFX12: ; %bb.0: ; %entry ; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -5053,7 +5258,6 @@ ; GFX6-IR-NEXT: [[NOT_NAN:%.*]] = fcmp ord float bitcast (i32 ptrtoint (ptr @gv to i32) to float), 0.000000e+00 ; GFX6-IR-NEXT: [[COND8:%.*]] = select i1 [[NOT_NAN]], float [[COND]], float bitcast (i32 ptrtoint (ptr @gv to i32) to float) ; GFX6-IR-NEXT: ret float [[COND8]] -; ; IR-FRACT-LABEL: define float @safe_math_fract_f32_swapped_edge_case_x_is_const( ; IR-FRACT-SAME: ) #[[ATTR0]] { ; IR-FRACT-NEXT: [[ENTRY:.*:]] @@ -5062,7 +5266,6 @@ ; IR-FRACT-NEXT: [[NOT_INF:%.*]] = fcmp une float [[X_FABS]], +inf ; IR-FRACT-NEXT: [[COND8:%.*]] = select i1 [[NOT_INF]], float [[COND]], float 0.000000e+00 ; IR-FRACT-NEXT: ret float [[COND8]] -; entry: %floor = call float @llvm.floor.f32(float bitcast (i32 ptrtoint (ptr @gv to i32) to float)) %sub = fsub float bitcast (i32 ptrtoint (ptr @gv to i32) to float), %floor @@ -5236,7 +5439,6 @@ ; GFX6-IR-NEXT: ret float [[COND8]] ; GFX6-IR: [[RET]]: ; GFX6-IR-NEXT: ret float [[MIN]] -; ; IR-FRACT-LABEL: define float @safe_math_fract_f32_swapped_edge_case_split_block( ; IR-FRACT-SAME: float [[X:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] { ; IR-FRACT-NEXT: [[FLOOR:%.*]] = call float @llvm.floor.f32(float [[X]]) @@ -5251,7 +5453,6 @@ ; IR-FRACT-NEXT: ret float [[COND8]] ; IR-FRACT: [[RET]]: ; IR-FRACT-NEXT: ret float [[MIN]] -; %floor = call float @llvm.floor.f32(float %x) %sub = fsub float %x, %floor %min = call float @llvm.minnum.f32(float %sub, float 0x3FEFFFFFE0000000) @@ -5358,7 +5559,6 @@ ; GFX6-IR-NEXT: [[NOT_NAN:%.*]] = fcmp ord <3 x float> [[X]], <float 0.000000e+00, float poison, float 0.000000e+00> ; GFX6-IR-NEXT: [[COND8:%.*]] = select <3 x i1> [[NOT_NAN]], <3 x float> [[COND]], <3 x float> [[X]] ; GFX6-IR-NEXT: ret <3 x float> [[COND8]] -; ; IR-FRACT-LABEL: define <3 x float> @safe_math_fract_f32_swapped_edge_case_vector( ; IR-FRACT-SAME: <3 x float> [[X:%.*]]) #[[ATTR0]] { ; IR-FRACT-NEXT: [[ENTRY:.*:]] @@ -5375,7 +5575,6 @@ ; IR-FRACT-NEXT: [[NOT_INF:%.*]] = fcmp une <3 x float> [[X_FABS]], <float +inf, float poison, float +inf> ; IR-FRACT-NEXT: [[COND8:%.*]] = select <3 x i1> [[NOT_INF]], <3 x float> [[COND]], <3 x float> <float 0.000000e+00, float poison, float 0.000000e+00> ; IR-FRACT-NEXT: ret <3 x float> [[COND8]] -; entry: %floor = call <3 x float> @llvm.floor.v3f32(<3 x float> %x) %sub = fsub <3 x float> %x, %floor @@ -5438,7 +5637,6 @@ ; IR-NEXT: [[NOT_NAN:%.*]] = fcmp ord float [[X]], 0.000000e+00 ; IR-NEXT: [[COND8:%.*]] = select i1 [[NOT_NAN]], float [[COND]], float [[X]] ; IR-NEXT: ret float [[COND8]] -; entry: %floor = call float @llvm.floor.f32(float %x) %sub = fsub float %x, %floor @@ -5537,7 +5735,6 @@ ; GFX6-IR-NEXT: [[NOT_INF:%.*]] = fcmp une float [[FABS_X]], +inf ; GFX6-IR-NEXT: [[COND6:%.*]] = select i1 [[NOT_INF]], float [[COND]], float 0.000000e+00 ; GFX6-IR-NEXT: ret float [[COND6]] -; ; IR-FRACT-LABEL: define float @fract_pat_fcmp_oge_select( ; IR-FRACT-SAME: float [[X:%.*]], ptr addrspace(1) [[IPTR:%.*]]) #[[ATTR0]] { ; IR-FRACT-NEXT: [[ENTRY:.*:]] @@ -5548,7 +5745,6 @@ ; IR-FRACT-NEXT: [[NOT_INF:%.*]] = fcmp une float [[FABS_X]], +inf ; IR-FRACT-NEXT: [[COND6:%.*]] = select i1 [[NOT_INF]], float [[COND]], float 0.000000e+00 ; IR-FRACT-NEXT: ret float [[COND6]] -; entry: %call = call float @llvm.floor.f32(float %x) store float %call, ptr addrspace(1) %iptr, align 4 @@ -5645,7 +5841,6 @@ ; GFX6-IR-NEXT: [[NOT_INF:%.*]] = fcmp une float [[FABS_X]], +inf ; GFX6-IR-NEXT: [[COND6:%.*]] = select i1 [[NOT_INF]], float [[COND]], float 0.000000e+00 ; GFX6-IR-NEXT: ret float [[COND6]] -; ; IR-FRACT-LABEL: define float @fract_pat_fcmp_ogt_select( ; IR-FRACT-SAME: float [[X:%.*]], ptr addrspace(1) [[IPTR:%.*]]) #[[ATTR0]] { ; IR-FRACT-NEXT: [[ENTRY:.*:]] @@ -5656,7 +5851,6 @@ ; IR-FRACT-NEXT: [[NOT_INF:%.*]] = fcmp une float [[FABS_X]], +inf ; IR-FRACT-NEXT: [[COND6:%.*]] = select i1 [[NOT_INF]], float [[COND]], float 0.000000e+00 ; IR-FRACT-NEXT: ret float [[COND6]] -; entry: %call = call float @llvm.floor.f32(float %x) store float %call, ptr addrspace(1) %iptr, align 4 @@ -5758,7 +5952,6 @@ ; IR-NEXT: [[NOT_INF:%.*]] = fcmp une float [[FABS_X]], +inf ; IR-NEXT: [[COND6:%.*]] = select i1 [[NOT_INF]], float [[COND]], float 0.000000e+00 ; IR-NEXT: ret float [[COND6]] -; entry: %call = call float @llvm.floor.f32(float noundef %x) store float %call, ptr addrspace(1) %iptr, align 4 @@ -5859,7 +6052,6 @@ ; IR-NEXT: [[NOT_INF:%.*]] = fcmp une float [[FABS_X]], +inf ; IR-NEXT: [[COND6:%.*]] = select i1 [[NOT_INF]], float [[COND]], float 0.000000e+00 ; IR-NEXT: ret float [[COND6]] -; entry: %call = call float @llvm.floor.f32(float noundef %x) store float %call, ptr addrspace(1) %iptr, align 4 @@ -5959,7 +6151,6 @@ ; GFX6-IR-NEXT: [[NOT_INF:%.*]] = fcmp une float [[FABS_X]], +inf ; GFX6-IR-NEXT: [[COND6:%.*]] = select i1 [[NOT_INF]], float [[MIN]], float 0.000000e+00 ; GFX6-IR-NEXT: ret float [[COND6]] -; ; IR-FRACT-LABEL: define float @fract_pat_minimum( ; IR-FRACT-SAME: float [[X:%.*]], ptr addrspace(1) [[IPTR:%.*]]) #[[ATTR0]] { ; IR-FRACT-NEXT: [[ENTRY:.*:]] @@ -5970,7 +6161,6 @@ ; IR-FRACT-NEXT: [[NOT_INF:%.*]] = fcmp une float [[FABS_X]], +inf ; IR-FRACT-NEXT: [[COND6:%.*]] = select i1 [[NOT_INF]], float [[MIN]], float 0.000000e+00 ; IR-FRACT-NEXT: ret float [[COND6]] -; entry: %call = call float @llvm.floor.f32(float %x) store float %call, ptr addrspace(1) %iptr, align 4 @@ -6025,12 +6215,10 @@ ; GFX6-IR-NEXT: [[OGE_MIN_CONST:%.*]] = fcmp oge float [[SUB_FLOOR]], f0x3F7FFFFF ; GFX6-IR-NEXT: [[SELECT:%.*]] = select i1 [[OGE_MIN_CONST]], float f0x3F7FFFFF, float [[SUB_FLOOR]] ; GFX6-IR-NEXT: ret float [[SELECT]] -; ; IR-FRACT-LABEL: define float @core_fract_pat_fcmp_oge_select( ; IR-FRACT-SAME: float [[X:%.*]]) #[[ATTR0]] { ; IR-FRACT-NEXT: [[SELECT:%.*]] = call float @llvm.amdgcn.fract.f32(float [[X]]) ; IR-FRACT-NEXT: ret float [[SELECT]] -; %floor = call float @llvm.floor.f32(float %x) %sub.floor = fsub float %x, %floor %oge.min.const = fcmp oge float %sub.floor, 0x3FEFFFFFE0000000 @@ -6083,12 +6271,10 @@ ; GFX6-IR-NEXT: [[SUB_FLOOR:%.*]] = fsub float [[X]], [[FLOOR]] ; GFX6-IR-NEXT: [[MIN:%.*]] = call float @llvm.minimum.f32(float [[SUB_FLOOR]], float f0x3F7FFFFF) ; GFX6-IR-NEXT: ret float [[MIN]] -; ; IR-FRACT-LABEL: define float @core_fract_pat_minimum( ; IR-FRACT-SAME: float [[X:%.*]]) #[[ATTR0]] { ; IR-FRACT-NEXT: [[MIN:%.*]] = call nnan float @llvm.amdgcn.fract.f32(float [[X]]) ; IR-FRACT-NEXT: ret float [[MIN]] -; %floor = call float @llvm.floor.f32(float %x) %sub.floor = fsub float %x, %floor %min = call float @llvm.minimum.f32(float %sub.floor, float 0x3FEFFFFFE0000000) @@ -6144,7 +6330,6 @@ ; GFX6-IR-NEXT: [[OGE_MIN_CONST:%.*]] = fcmp oge <2 x float> [[SUB_FLOOR]], <float f0x3F7FFFFF, float poison> ; GFX6-IR-NEXT: [[SELECT:%.*]] = select <2 x i1> [[OGE_MIN_CONST]], <2 x float> <float f0x3F7FFFFF, float poison>, <2 x float> [[SUB_FLOOR]] ; GFX6-IR-NEXT: ret <2 x float> [[SELECT]] -; ; IR-FRACT-LABEL: define <2 x float> @core_fract_pat_fcmp_oge_select_v2f32( ; IR-FRACT-SAME: <2 x float> [[X:%.*]]) #[[ATTR0]] { ; IR-FRACT-NEXT: [[TMP1:%.*]] = extractelement <2 x float> [[X]], i64 0 @@ -6154,7 +6339,6 @@ ; IR-FRACT-NEXT: [[TMP5:%.*]] = insertelement <2 x float> poison, float [[TMP3]], i64 0 ; IR-FRACT-NEXT: [[SELECT:%.*]] = insertelement <2 x float> [[TMP5]], float [[TMP4]], i64 1 ; IR-FRACT-NEXT: ret <2 x float> [[SELECT]] -; %floor = call <2 x float> @llvm.floor.v2f32(<2 x float> %x) %sub.floor = fsub <2 x float> %x, %floor %oge.min.const = fcmp oge <2 x float> %sub.floor, <float 0x3FEFFFFFE0000000, float poison> @@ -6212,7 +6396,6 @@ ; GFX6-IR-NEXT: [[SUB_FLOOR:%.*]] = fsub <2 x float> [[X]], [[FLOOR]] ; GFX6-IR-NEXT: [[MIN:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[SUB_FLOOR]], <2 x float> <float f0x3F7FFFFF, float poison>) ; GFX6-IR-NEXT: ret <2 x float> [[MIN]] -; ; IR-FRACT-LABEL: define <2 x float> @core_fract_pat_minimum_v2f32( ; IR-FRACT-SAME: <2 x float> [[X:%.*]]) #[[ATTR0]] { ; IR-FRACT-NEXT: [[TMP1:%.*]] = extractelement <2 x float> [[X]], i64 0 @@ -6222,7 +6405,6 @@ ; IR-FRACT-NEXT: [[TMP5:%.*]] = insertelement <2 x float> poison, float [[TMP3]], i64 0 ; IR-FRACT-NEXT: [[MIN:%.*]] = insertelement <2 x float> [[TMP5]], float [[TMP4]], i64 1 ; IR-FRACT-NEXT: ret <2 x float> [[MIN]] -; %floor = call <2 x float> @llvm.floor.v2f32(<2 x float> %x) %sub.floor = fsub <2 x float> %x, %floor %min = call <2 x float> @llvm.minimum.v2f32(<2 x float> %sub.floor, <2 x float> <float 0x3FEFFFFFE0000000, float poison>)
diff --git a/llvm/test/CodeGen/AMDGPU/freeze.ll b/llvm/test/CodeGen/AMDGPU/freeze.ll index 5d520d8..b3049d9 100644 --- a/llvm/test/CodeGen/AMDGPU/freeze.ll +++ b/llvm/test/CodeGen/AMDGPU/freeze.ll
@@ -16227,11 +16227,11 @@ ; GFX11-SDAG-TRUE16-NEXT: global_load_b64 v[4:5], v[0:1], off ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v5 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4 ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 1, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l ; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 3 ; GFX11-SDAG-TRUE16-NEXT: global_store_b8 v[2:3], v0, off ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -16508,14 +16508,14 @@ ; GFX11-SDAG-TRUE16-NEXT: global_load_b96 v[4:6], v[0:1], off ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v5 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, 0, v4 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v6 -; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 2, v4.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 1, s0 +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 2, v1.l ; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h ; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 7 ; GFX11-SDAG-TRUE16-NEXT: global_store_b8 v[2:3], v0, off @@ -16847,19 +16847,18 @@ ; GFX11-SDAG-TRUE16-NEXT: global_load_b128 v[4:7], v[0:1], off ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v5 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v6 -; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, 0, v4 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s1, 0, v6 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 1, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s1 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 2, v4.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 3, v1.l +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 2, v1.l +; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 1, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 3, v0.h ; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h ; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 15 ; GFX11-SDAG-TRUE16-NEXT: global_store_b8 v[2:3], v0, off
diff --git a/llvm/test/CodeGen/AMDGPU/fshl-illegal-types.ll b/llvm/test/CodeGen/AMDGPU/fshl-illegal-types.ll index 64c01fa..e9394cdb 100644 --- a/llvm/test/CodeGen/AMDGPU/fshl-illegal-types.ll +++ b/llvm/test/CodeGen/AMDGPU/fshl-illegal-types.ll
@@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 ; RUN: llc -global-isel=0 -mtriple=amdgpu6.00 < %s | FileCheck %s --implicit-check-not=s_swappc_b64 ; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 < %s | FileCheck %s --implicit-check-not=s_swappc_b64 @@ -12,36 +13,28 @@ declare i127 @llvm.fshl.i127(i127, i127, i127) define i65 @fshl_i65(i65 %amt) { -; CHECK-LABEL: fshl_i65: -; CHECK: s_setpc_b64 %result = call i65 @llvm.fshl.i65(i65 1, i65 0, i65 %amt) ret i65 %result } define i66 @fshl_i66(i66 %amt) { -; CHECK-LABEL: fshl_i66: -; CHECK: s_setpc_b64 %result = call i66 @llvm.fshl.i66(i66 1, i66 0, i66 %amt) ret i66 %result } define i67 @fshl_i67(i67 %amt) { -; CHECK-LABEL: fshl_i67: -; CHECK: s_setpc_b64 %result = call i67 @llvm.fshl.i67(i67 1, i67 0, i67 %amt) ret i67 %result } define i68 @fshl_i68(i68 %amt) { -; CHECK-LABEL: fshl_i68: -; CHECK: s_setpc_b64 %result = call i68 @llvm.fshl.i68(i68 1, i68 0, i68 %amt) ret i68 %result } define i127 @fshl_i127(i127 %amt) { -; CHECK-LABEL: fshl_i127: -; CHECK: s_setpc_b64 %result = call i127 @llvm.fshl.i127(i127 1, i127 0, i127 %amt) ret i127 %result } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/function-esm2-prologue-epilogue.ll b/llvm/test/CodeGen/AMDGPU/function-esm2-prologue-epilogue.ll index 87b60b8..33eaec74 100644 --- a/llvm/test/CodeGen/AMDGPU/function-esm2-prologue-epilogue.ll +++ b/llvm/test/CodeGen/AMDGPU/function-esm2-prologue-epilogue.ll
@@ -15,7 +15,7 @@ ; GFX12-NEXT: s_wait_kmcnt 0x0 ; GFX12-NEXT: v_perm_b32 v0, 0, v0, 0xc0c0001 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-NEXT: v_cvt_f32_f16_e32 v0, v0 +; GFX12-NEXT: v_cvt_f32_f16_e32 v0, v0.l ; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-ESM-LABEL: missing_truncate_promote_bswap: @@ -28,7 +28,7 @@ ; GFX12-ESM-NEXT: s_wait_kmcnt 0x0 ; GFX12-ESM-NEXT: v_perm_b32 v0, 0, v0, 0xc0c0001 ; GFX12-ESM-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-ESM-NEXT: v_cvt_f32_f16_e32 v0, v0 +; GFX12-ESM-NEXT: v_cvt_f32_f16_e32 v0, v0.l ; GFX12-ESM-NEXT: s_wait_alu depctr_va_vdst(0) ; GFX12-ESM-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_SCHED_MODE, 0, 2), 0 ; GFX12-ESM-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll index d32896d..5c7a806 100644 --- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll +++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -13717,114 +13717,58 @@ ; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5 ; GFX1250-NEXT: s_set_pc_i64 s[30:31] ; -; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB54_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB54_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB54_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -13869,106 +13813,54 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB54_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB54_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -14262,119 +14154,61 @@ ; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5 ; GFX1250-NEXT: s_set_pc_i64 s[30:31] ; -; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB55_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB55_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB55_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -14421,110 +14255,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB55_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB55_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB55_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -14827,119 +14607,61 @@ ; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5 ; GFX1250-NEXT: s_set_pc_i64 s[30:31] ; -; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB56_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB56_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB56_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -14987,110 +14709,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB56_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB56_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -15390,111 +15058,56 @@ ; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX1250-NEXT: s_set_pc_i64 s[30:31] ; -; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX12-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB57_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX12-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB57_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v6, v3 +; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: v_add_f32_e32 v3, v3, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-NEXT: v_mov_b32_e32 v4, v3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB57_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -15538,103 +15151,52 @@ ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB57_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB57_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v6, v3 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: v_add_f32_e32 v3, v3, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB57_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -16830,33 +16392,32 @@ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046 +; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) @@ -16864,7 +16425,7 @@ ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB60_1 ; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: @@ -16949,41 +16510,40 @@ ; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046 +; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB60_1 ; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: @@ -17627,121 +17187,62 @@ ; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5 ; GFX1250-NEXT: s_set_pc_i64 s[30:31] ; -; GFX12-TRUE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB62_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB62_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB62_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB62_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB62_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: global_wb scope:SCOPE_SYS +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_SYS +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB62_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -17788,110 +17289,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB62_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB62_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB62_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB62_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB62_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB62_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll index 3f674bb..445191d 100644 --- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll +++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -8677,114 +8677,58 @@ ; -------------------------------------------------------------------- define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB36_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -8829,106 +8773,54 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_max_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB36_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -9180,119 +9072,61 @@ } define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB37_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -9339,110 +9173,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_max_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB37_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -9702,119 +9482,61 @@ } define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB38_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -9862,110 +9584,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_max_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB38_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -10225,111 +9893,56 @@ } define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v6, v3 +; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-NEXT: v_mov_b32_e32 v4, v3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB39_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -10373,103 +9986,52 @@ ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v2 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v6, v3 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: v_max_f32_e32 v3, v3, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB39_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -11522,33 +11084,32 @@ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046 +; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v3 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) @@ -11556,7 +11117,7 @@ ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1 ; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: @@ -11641,41 +11202,40 @@ ; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046 +; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1 ; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: @@ -12247,121 +11807,62 @@ } define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: global_wb scope:SCOPE_SYS +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_SYS +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB44_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -12408,110 +11909,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_max_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB44_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll index 7c09a40..8b34afe 100644 --- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll +++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -8677,114 +8677,58 @@ ; -------------------------------------------------------------------- define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB36_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -8829,106 +8773,54 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_min_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB36_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -9180,119 +9072,61 @@ } define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB37_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -9339,110 +9173,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_min_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB37_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -9702,119 +9482,61 @@ } define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB38_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -9862,110 +9584,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_min_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB38_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -10225,111 +9893,56 @@ } define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v6, v3 +; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: v_min_num_f32_e32 v3, v3, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-NEXT: v_mov_b32_e32 v4, v3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB39_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -10373,103 +9986,52 @@ ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v2 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v6, v3 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: v_min_f32_e32 v3, v3, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB39_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0: @@ -11522,33 +11084,32 @@ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046 +; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v3 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) @@ -11556,7 +11117,7 @@ ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1 ; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: @@ -11641,41 +11202,40 @@ ; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046 +; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1 ; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory: @@ -12247,121 +11807,62 @@ } define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: global_wb scope:SCOPE_SYS +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_SYS +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB44_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX942: ; %bb.0: @@ -12408,110 +11909,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_min_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB44_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll index f55b03c..ef314c8 100644 --- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll +++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
@@ -9223,114 +9223,58 @@ ; -------------------------------------------------------------------- define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB32_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB32_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB32_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fsub_ret_bf16: ; GFX942: ; %bb.0: @@ -9375,106 +9319,54 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB32_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB32_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB32_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16: ; GFX10: ; %bb.0: @@ -9724,119 +9616,61 @@ } define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB33_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB33_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB33_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos: ; GFX942: ; %bb.0: @@ -9883,110 +9717,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB33_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB33_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB33_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos: ; GFX10: ; %bb.0: @@ -10244,119 +10024,61 @@ } define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1) %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB34_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB34_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg: ; GFX942: ; %bb.0: @@ -10404,110 +10126,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB34_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB34_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg: ; GFX10: ; %bb.0: @@ -10765,111 +10433,56 @@ } define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB35_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_noret_bf16: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB35_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_agent_atomic_fsub_noret_bf16: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v6, v3 +; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: v_sub_f32_e32 v3, v3, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_DEV +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-NEXT: v_mov_b32_e32 v4, v3 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB35_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_agent_atomic_fsub_noret_bf16: ; GFX942: ; %bb.0: @@ -10913,103 +10526,52 @@ ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB35_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_noret_bf16: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB35_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_agent_atomic_fsub_noret_bf16: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_not_b32_e32 v6, v3 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: v_sub_f32_e32 v3, v3, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo +; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3 +; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB35_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_agent_atomic_fsub_noret_bf16: ; GFX10: ; %bb.0: @@ -12056,33 +11618,32 @@ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046 +; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) @@ -12090,7 +11651,7 @@ ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1 ; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4: @@ -12175,41 +11736,40 @@ ; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046 +; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1 ; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4: @@ -12777,121 +12337,62 @@ } define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 { -; GFX12-TRUE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos: -; GFX12-TRUE16: ; %bb.0: -; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start -; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB40_1 -; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX12-FAKE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos: -; GFX12-FAKE16: ; %bb.0: -; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 -; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX12-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start -; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0 -; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) -; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB40_1 -; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX12-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-NEXT: s_mov_b32 s0, 0 +; GFX12-NEXT: global_load_b32 v5, v[0:1], off +; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_not_b32_e32 v4, v4 +; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start +; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_mov_b32_e32 v6, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX12-NEXT: global_wb scope:SCOPE_SYS +; GFX12-NEXT: s_wait_storecnt 0x0 +; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_inv scope:SCOPE_SYS +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0) +; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: s_cbranch_execnz .LBB40_1 +; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX12-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos: ; GFX942: ; %bb.0: @@ -12938,110 +12439,56 @@ ; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-TRUE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX11-TRUE16-NEXT: .p2align 6 -; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start -; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5 -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_gl1_inv -; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB40_1 -; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5 -; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-FAKE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3 -; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0 -; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4 -; GFX11-FAKE16-NEXT: .p2align 6 -; GFX11-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start -; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 -; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5 -; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc -; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-FAKE16-NEXT: buffer_gl1_inv -; GFX11-FAKE16-NEXT: buffer_gl0_inv -; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 -; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0 -; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB40_1 -; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5 -; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: global_load_b32 v5, v[0:1], off +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff +; GFX11-NEXT: v_not_b32_e32 v4, v4 +; GFX11-NEXT: .p2align 6 +; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start +; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mov_b32_e32 v6, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2 +; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5 +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6 +; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execnz .LBB40_1 +; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos: ; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/idot4s.ll b/llvm/test/CodeGen/AMDGPU/idot4s.ll index 31622c3..332281b 100644 --- a/llvm/test/CodeGen/AMDGPU/idot4s.ll +++ b/llvm/test/CodeGen/AMDGPU/idot4s.ll
@@ -1138,45 +1138,45 @@ ; GFX11-DL-TRUE16-LABEL: idot4_acc16_vecMul: ; GFX11-DL-TRUE16: ; %bb.0: ; %entry ; GFX11-DL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 -; GFX11-DL-TRUE16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v0 +; GFX11-DL-TRUE16-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0x3ff, v0 ; GFX11-DL-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 ; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-DL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX11-DL-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-DL-TRUE16-NEXT: s_clause 0x1 -; GFX11-DL-TRUE16-NEXT: global_load_b32 v1, v0, s[0:1] -; GFX11-DL-TRUE16-NEXT: global_load_b32 v2, v0, s[2:3] -; GFX11-DL-TRUE16-NEXT: global_load_d16_b16 v0, v3, s[4:5] +; GFX11-DL-TRUE16-NEXT: global_load_b32 v3, v0, s[0:1] +; GFX11-DL-TRUE16-NEXT: global_load_b32 v4, v0, s[2:3] +; GFX11-DL-TRUE16-NEXT: global_load_d16_b16 v0, v5, s[4:5] ; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(2) -; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v1, 0, 8 +; GFX11-DL-TRUE16-NEXT: v_ashrrev_i16 v2.l, 8, v3.l ; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1) -; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 8 -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1 -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2 -; GFX11-DL-TRUE16-NEXT: v_ashrrev_i16 v1.h, 8, v1.l -; GFX11-DL-TRUE16-NEXT: v_ashrrev_i16 v2.h, 8, v2.l -; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.l -; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l -; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v5, v7, 0, 8 -; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v4, 0, 8 -; GFX11-DL-TRUE16-NEXT: v_ashrrev_i16 v4.h, 8, v7.l -; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX11-DL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v2 -; GFX11-DL-TRUE16-NEXT: v_ashrrev_i16 v2.h, 8, v4.l -; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l -; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1 +; GFX11-DL-TRUE16-NEXT: v_ashrrev_i16 v1.l, 8, v4.l +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3 +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v4 +; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8 +; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v4, v4, 0, 8 +; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l +; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX11-DL-TRUE16-NEXT: v_ashrrev_i16 v1.l, 8, v7.l +; GFX11-DL-TRUE16-NEXT: v_ashrrev_i16 v2.l, 8, v6.l +; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-DL-TRUE16-NEXT: v_pk_mul_lo_u16 v3, v3, v4 +; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v4, v6, 0, 8 +; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v7, 0, 8 +; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3 ; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v1.l, v0.l +; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v3.l, v0.l ; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-DL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v2, v4 -; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v5.l -; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1 +; GFX11-DL-TRUE16-NEXT: v_pk_mul_lo_u16 v2, v4, v6 ; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l -; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2 ; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l -; GFX11-DL-TRUE16-NEXT: global_store_b16 v3, v0, s[4:5] +; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l +; GFX11-DL-TRUE16-NEXT: global_store_b16 v5, v0, s[4:5] ; GFX11-DL-TRUE16-NEXT: s_endpgm ; ; GFX11-DL-FAKE16-LABEL: idot4_acc16_vecMul: @@ -3369,34 +3369,31 @@ ; GFX11-DL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX11-DL-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-DL-TRUE16-NEXT: s_clause 0x1 -; GFX11-DL-TRUE16-NEXT: global_load_b32 v3, v0, s[0:1] -; GFX11-DL-TRUE16-NEXT: global_load_b32 v4, v0, s[2:3] +; GFX11-DL-TRUE16-NEXT: global_load_b32 v1, v0, s[0:1] +; GFX11-DL-TRUE16-NEXT: global_load_b32 v2, v0, s[2:3] ; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1) -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 8, v3 +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 8, v1 ; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4 -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3 -; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v3, 0, 8 -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v4 -; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v0, 0, 8 -; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l -; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.l -; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8 -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 24, v4 -; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v3 -; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l -; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-DL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l -; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v7.l -; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v2.l, v0.h, v0.l -; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v6, 0, 8 -; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v2 +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2 +; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v1, 0, 8 +; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v4, v0, 0, 8 +; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v3.l +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1 +; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l +; GFX11-DL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v4.l +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 24, v1 +; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8 +; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v5.l +; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v6.l, v0.h, v0.l +; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v4, v4, 0, 8 ; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v1.l, v3.l, v0.l -; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 24, v2 ; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v1.l, v4.l, v0.l +; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v4.l, v1.l, v0.l ; GFX11-DL-TRUE16-NEXT: v_mov_b32_e32 v1, 0 ; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16 ; GFX11-DL-TRUE16-NEXT: global_store_b32 v1, v0, s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll index 5e6c34c..34147bb 100644 --- a/llvm/test/CodeGen/AMDGPU/idot4u.ll +++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -1647,34 +1647,33 @@ ; GFX11-DL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 ; GFX11-DL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX11-DL-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 -; GFX11-DL-TRUE16-NEXT: v_mov_b32_e32 v6, 0 +; GFX11-DL-TRUE16-NEXT: v_mov_b32_e32 v4, 0 ; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-DL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX11-DL-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-DL-TRUE16-NEXT: s_clause 0x1 -; GFX11-DL-TRUE16-NEXT: global_load_b32 v4, v0, s[0:1] -; GFX11-DL-TRUE16-NEXT: global_load_b32 v5, v0, s[2:3] -; GFX11-DL-TRUE16-NEXT: global_load_d16_b16 v0, v6, s[4:5] +; GFX11-DL-TRUE16-NEXT: global_load_b32 v2, v0, s[0:1] +; GFX11-DL-TRUE16-NEXT: global_load_b32 v3, v0, s[2:3] +; GFX11-DL-TRUE16-NEXT: global_load_d16_b16 v0, v4, s[4:5] ; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(2) -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4 +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2 ; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1) -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v5 -; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8 -; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v7, v5, 0, 8 +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v3 +; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v3, 0, 8 ; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l -; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l -; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l -; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l -; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v5.l +; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 8 +; GFX11-DL-TRUE16-NEXT: v_perm_b32 v2, v2, v2, 0xc0c0302 ; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v0.l -; GFX11-DL-TRUE16-NEXT: v_perm_b32 v1, v5, v5, 0xc0c0302 -; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v2.l, v3.l, v0.l -; GFX11-DL-TRUE16-NEXT: v_perm_b32 v2, v4, v4, 0xc0c0302 +; GFX11-DL-TRUE16-NEXT: v_perm_b32 v1, v3, v3, 0xc0c0302 ; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v5.l, v6.l, v0.l ; GFX11-DL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l +; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-DL-TRUE16-NEXT: v_dot4_u32_u8 v0, v2, v1, v0 -; GFX11-DL-TRUE16-NEXT: global_store_b16 v6, v0, s[4:5] +; GFX11-DL-TRUE16-NEXT: global_store_b16 v4, v0, s[4:5] ; GFX11-DL-TRUE16-NEXT: s_endpgm ; ; GFX11-DL-FAKE16-LABEL: notdot4_mixedtypes: @@ -1936,41 +1935,39 @@ ; GFX11-DL-TRUE16-LABEL: notdot4_mixedtypes2: ; GFX11-DL-TRUE16: ; %bb.0: ; %entry ; GFX11-DL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 -; GFX11-DL-TRUE16-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0x3ff, v0 +; GFX11-DL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX11-DL-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 -; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-DL-TRUE16-NEXT: v_mov_b32_e32 v4, 0 +; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-DL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX11-DL-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-DL-TRUE16-NEXT: s_clause 0x1 +; GFX11-DL-TRUE16-NEXT: global_load_b32 v2, v0, s[0:1] ; GFX11-DL-TRUE16-NEXT: global_load_b32 v3, v0, s[2:3] -; GFX11-DL-TRUE16-NEXT: global_load_b32 v4, v0, s[0:1] -; GFX11-DL-TRUE16-NEXT: global_load_d16_b16 v0, v5, s[4:5] +; GFX11-DL-TRUE16-NEXT: global_load_d16_b16 v0, v4, s[4:5] ; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(2) -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3 +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2 ; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1) -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v4 -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4 -; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v7, v4, 0, 8 -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3 -; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v3 +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2 +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3 +; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v8, v2, 0, 8 ; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l -; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l -; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8 -; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.l +; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l +; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8 +; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l ; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3 -; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v0.l -; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v8.l -; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l ; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v2.l, v1.h, v0.l +; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v5.l, v0.l +; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v5, v6, 0, 8 +; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v7.l +; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v8.l, v1.l, v0.l +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 24, v2 ; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 8 -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v4 -; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v1.l, v0.h, v0.l ; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v3.l, v1.l, v0.l -; GFX11-DL-TRUE16-NEXT: global_store_b16 v5, v0, s[4:5] +; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v5.l, v0.h, v0.l +; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v1.l, v2.l, v0.l +; GFX11-DL-TRUE16-NEXT: global_store_b16 v4, v0, s[4:5] ; GFX11-DL-TRUE16-NEXT: s_endpgm ; ; GFX11-DL-FAKE16-LABEL: notdot4_mixedtypes2:
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll index f51d7f5..9267f43 100644 --- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll +++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
@@ -853,47 +853,95 @@ } define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat %b) { - ; DAGISEL-GFX11-WF32-LABEL: name: amdgpu_cs_chain_cc_bfloat - ; DAGISEL-GFX11-WF32: bb.0 (%ir-block.0): - ; DAGISEL-GFX11-WF32-NEXT: liveins: $sgpr0, $vgpr8 - ; DAGISEL-GFX11-WF32-NEXT: {{ $}} - ; DAGISEL-GFX11-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8 - ; DAGISEL-GFX11-WF32-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0 - ; DAGISEL-GFX11-WF32-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec - ; DAGISEL-GFX11-WF32-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc - ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec - ; DAGISEL-GFX11-WF32-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec - ; DAGISEL-GFX11-WF32-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767 - ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec - ; DAGISEL-GFX11-WF32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304 - ; DAGISEL-GFX11-WF32-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec - ; DAGISEL-GFX11-WF32-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec - ; DAGISEL-GFX11-WF32-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec - ; DAGISEL-GFX11-WF32-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF - ; DAGISEL-GFX11-WF32-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]] - ; DAGISEL-GFX11-WF32-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`) - ; DAGISEL-GFX11-WF32-NEXT: S_ENDPGM 0 + ; DAGISEL-GFX11-WF32-TRUE16-LABEL: name: amdgpu_cs_chain_cc_bfloat + ; DAGISEL-GFX11-WF32-TRUE16: bb.0 (%ir-block.0): + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: liveins: $sgpr0, $vgpr8 + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: {{ $}} + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8 + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0 + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, killed [[DEF]], %subreg.hi16 + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, killed [[REG_SEQUENCE]], implicit $exec + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767 + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304 + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[DEF2:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF2]] + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`) + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: S_ENDPGM 0 ; - ; DAGISEL-GFX11-WF64-LABEL: name: amdgpu_cs_chain_cc_bfloat - ; DAGISEL-GFX11-WF64: bb.0 (%ir-block.0): - ; DAGISEL-GFX11-WF64-NEXT: liveins: $sgpr0, $vgpr8 - ; DAGISEL-GFX11-WF64-NEXT: {{ $}} - ; DAGISEL-GFX11-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8 - ; DAGISEL-GFX11-WF64-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0 - ; DAGISEL-GFX11-WF64-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec - ; DAGISEL-GFX11-WF64-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc - ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec - ; DAGISEL-GFX11-WF64-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec - ; DAGISEL-GFX11-WF64-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767 - ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec - ; DAGISEL-GFX11-WF64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304 - ; DAGISEL-GFX11-WF64-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec - ; DAGISEL-GFX11-WF64-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_64_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec - ; DAGISEL-GFX11-WF64-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec - ; DAGISEL-GFX11-WF64-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF - ; DAGISEL-GFX11-WF64-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]] - ; DAGISEL-GFX11-WF64-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`) - ; DAGISEL-GFX11-WF64-NEXT: S_ENDPGM 0 + ; DAGISEL-GFX11-WF32-FAKE16-LABEL: name: amdgpu_cs_chain_cc_bfloat + ; DAGISEL-GFX11-WF32-FAKE16: bb.0 (%ir-block.0): + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: liveins: $sgpr0, $vgpr8 + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: {{ $}} + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8 + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0 + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767 + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304 + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]] + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`) + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: S_ENDPGM 0 + ; + ; DAGISEL-GFX11-WF64-TRUE16-LABEL: name: amdgpu_cs_chain_cc_bfloat + ; DAGISEL-GFX11-WF64-TRUE16: bb.0 (%ir-block.0): + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: liveins: $sgpr0, $vgpr8 + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: {{ $}} + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8 + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0 + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, killed [[DEF]], %subreg.hi16 + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, killed [[REG_SEQUENCE]], implicit $exec + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767 + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304 + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_64_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[DEF2:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF2]] + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`) + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: S_ENDPGM 0 + ; + ; DAGISEL-GFX11-WF64-FAKE16-LABEL: name: amdgpu_cs_chain_cc_bfloat + ; DAGISEL-GFX11-WF64-FAKE16: bb.0 (%ir-block.0): + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: liveins: $sgpr0, $vgpr8 + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: {{ $}} + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8 + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0 + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767 + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304 + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_64_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]] + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`) + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: S_ENDPGM 0 ; ; DAGISEL-GFX10-WF32-LABEL: name: amdgpu_cs_chain_cc_bfloat ; DAGISEL-GFX10-WF32: bb.0 (%ir-block.0):
diff --git a/llvm/test/CodeGen/AMDGPU/literals.ll b/llvm/test/CodeGen/AMDGPU/literals.ll index 0e6400e..2d4a960 100644 --- a/llvm/test/CodeGen/AMDGPU/literals.ll +++ b/llvm/test/CodeGen/AMDGPU/literals.ll
@@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 ; RUN: llc < %s -mtriple=r600 -mcpu=redwood | FileCheck %s ; Test using an integer literal constant. @@ -11,6 +12,16 @@ ; CHECK-NEXT: ADD_INT * {{\** *}}T{{[0-9]\.[XYZW]}}, KC0[2].Z, literal.y ; CHECK-NEXT: 5 define amdgpu_kernel void @i32_literal(ptr addrspace(1) %out, i32 %in) { +; CHECK-LABEL: i32_literal: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; CHECK-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; CHECK-NEXT: CF_END +; CHECK-NEXT: PAD +; CHECK-NEXT: ALU clause starting at 4: +; CHECK-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; CHECK-NEXT: ADD_INT * T1.X, KC0[2].Z, literal.y, +; CHECK-NEXT: 2(2.802597e-45), 5(7.006492e-45) entry: %0 = add i32 5, %in store i32 %0, ptr addrspace(1) %out @@ -28,6 +39,16 @@ ; CHECK-NEXT: ADD * {{\** *}}T{{[0-9]\.[XYZW]}}, KC0[2].Z, literal.y ; CHECK-NEXT: 1084227584(5.0 define amdgpu_kernel void @float_literal(ptr addrspace(1) %out, float %in) { +; CHECK-LABEL: float_literal: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[] +; CHECK-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; CHECK-NEXT: CF_END +; CHECK-NEXT: PAD +; CHECK-NEXT: ALU clause starting at 4: +; CHECK-NEXT: LSHR T0.X, KC0[2].Y, literal.x, +; CHECK-NEXT: ADD * T1.X, KC0[2].Z, literal.y, +; CHECK-NEXT: 2(2.802597e-45), 1084227584(5.000000e+00) entry: %0 = fadd float 5.0, %in store float %0, ptr addrspace(1) %out @@ -42,6 +63,19 @@ ; CHECK-NEXT: MOV {{\** *}}T[[GPR]].W, 0.0 define amdgpu_kernel void @inline_literal_reg_sequence(ptr addrspace(1) %out) { +; CHECK-LABEL: inline_literal_reg_sequence: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: ALU 5, @4, KC0[CB0:0-32], KC1[] +; CHECK-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1 +; CHECK-NEXT: CF_END +; CHECK-NEXT: PAD +; CHECK-NEXT: ALU clause starting at 4: +; CHECK-NEXT: MOV T0.X, 0.0, +; CHECK-NEXT: MOV T0.Y, 0.0, +; CHECK-NEXT: MOV T0.Z, 0.0, +; CHECK-NEXT: MOV T0.W, 0.0, +; CHECK-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; CHECK-NEXT: 2(2.802597e-45), 0(0.000000e+00) entry: store <4 x i32> <i32 0, i32 0, i32 0, i32 0>, ptr addrspace(1) %out ret void @@ -53,6 +87,19 @@ ; CHECK-NEXT: DOT4 T[[GPR]].Z (MASKED), 1.0 ; CHECK-NEXT: DOT4 * T[[GPR]].W (MASKED), 1.0 define amdgpu_kernel void @inline_literal_dot4(ptr addrspace(1) %out) { +; CHECK-LABEL: inline_literal_dot4: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: ALU 5, @4, KC0[CB0:0-32], KC1[] +; CHECK-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1 +; CHECK-NEXT: CF_END +; CHECK-NEXT: PAD +; CHECK-NEXT: ALU clause starting at 4: +; CHECK-NEXT: LSHR * T0.X, KC0[2].Y, literal.x, +; CHECK-NEXT: 2(2.802597e-45), 0(0.000000e+00) +; CHECK-NEXT: DOT4 T1.X, 1.0, 1.0, +; CHECK-NEXT: DOT4 T1.Y (MASKED), 1.0, 1.0, +; CHECK-NEXT: DOT4 T1.Z (MASKED), 1.0, 1.0, +; CHECK-NEXT: DOT4 * T1.W (MASKED), 1.0, 1.0, entry: %0 = call float @llvm.r600.dot4(<4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>) store float %0, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll index bb7ed3b5..e8ca597 100644 --- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll +++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 ; UNSUPPORTED: expensive_checks ; RUN: llc -O0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \ ; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O0 %s @@ -1480,5 +1481,29 @@ ; GCN-O3-NEXT: Free MachineFunction define void @empty() { +; GCN-O0-LABEL: empty: +; GCN-O0: ; %bb.0: +; GCN-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-O0-NEXT: s_setpc_b64 s[30:31] +; +; GCN-O1-LABEL: empty: +; GCN-O1: ; %bb.0: +; GCN-O1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-O1-NEXT: s_setpc_b64 s[30:31] +; +; GCN-O1-OPTS-LABEL: empty: +; GCN-O1-OPTS: ; %bb.0: +; GCN-O1-OPTS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-O1-OPTS-NEXT: s_setpc_b64 s[30:31] +; +; GCN-O2-LABEL: empty: +; GCN-O2: ; %bb.0: +; GCN-O2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-O2-NEXT: s_setpc_b64 s[30:31] +; +; GCN-O3-LABEL: empty: +; GCN-O3: ; %bb.0: +; GCN-O3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-O3-NEXT: s_setpc_b64 s[30:31] ret void }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.d16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.d16.dim.ll index 1aaa2a0..5018c43 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.d16.dim.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.d16.dim.ll
@@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 ; RUN: llc < %s -mtriple=amdgpu8.02 | FileCheck -check-prefixes=GCN,UNPACKED,GFX89 %s ; RUN: llc < %s -mtriple=amdgpu8.10 | FileCheck -check-prefixes=GCN,GFX81,GFX89 %s ; RUN: llc < %s -mtriple=amdgpu9.00 | FileCheck -check-prefixes=GCN,PACKED,GFX89 %s @@ -11,6 +12,35 @@ ; GFX10: image_load v0, v[0:1], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm d16{{$}} ; GFX12PLUS: image_load v0, [v0, v1], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D d16 define amdgpu_ps half @image_load_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) { +; UNPACKED-LABEL: image_load_f16: +; UNPACKED: ; %bb.0: ; %main_body +; UNPACKED-NEXT: image_load v0, v[0:1], s[0:7] dmask:0x1 unorm d16 +; UNPACKED-NEXT: s_waitcnt vmcnt(0) +; UNPACKED-NEXT: ; return to shader part epilog +; +; GFX81-LABEL: image_load_f16: +; GFX81: ; %bb.0: ; %main_body +; GFX81-NEXT: image_load v0, v[0:1], s[0:7] dmask:0x1 unorm d16 +; GFX81-NEXT: s_waitcnt vmcnt(0) +; GFX81-NEXT: ; return to shader part epilog +; +; PACKED-LABEL: image_load_f16: +; PACKED: ; %bb.0: ; %main_body +; PACKED-NEXT: image_load v0, v[0:1], s[0:7] dmask:0x1 unorm d16 +; PACKED-NEXT: s_waitcnt vmcnt(0) +; PACKED-NEXT: ; return to shader part epilog +; +; GFX10-LABEL: image_load_f16: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: image_load v0, v[0:1], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm d16 +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: ; return to shader part epilog +; +; GFX12PLUS-LABEL: image_load_f16: +; GFX12PLUS: ; %bb.0: ; %main_body +; GFX12PLUS-NEXT: image_load v0, [v0, v1], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D d16 +; GFX12PLUS-NEXT: s_wait_loadcnt 0x0 +; GFX12PLUS-NEXT: ; return to shader part epilog main_body: %tex = call half @llvm.amdgcn.image.load.2d.f16.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0) ret half %tex @@ -23,6 +53,37 @@ ; GFX10: image_load v0, v[0:1], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm d16{{$}} ; GFX12PLUS: image_load v0, [v0, v1], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D d16 define amdgpu_ps float @image_load_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) { +; UNPACKED-LABEL: image_load_v2f16: +; UNPACKED: ; %bb.0: ; %main_body +; UNPACKED-NEXT: image_load v[0:1], v[0:1], s[0:7] dmask:0x3 unorm d16 +; UNPACKED-NEXT: s_waitcnt vmcnt(0) +; UNPACKED-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; UNPACKED-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; UNPACKED-NEXT: ; return to shader part epilog +; +; GFX81-LABEL: image_load_v2f16: +; GFX81: ; %bb.0: ; %main_body +; GFX81-NEXT: image_load v0, v[0:1], s[0:7] dmask:0x3 unorm d16 +; GFX81-NEXT: s_waitcnt vmcnt(0) +; GFX81-NEXT: ; return to shader part epilog +; +; PACKED-LABEL: image_load_v2f16: +; PACKED: ; %bb.0: ; %main_body +; PACKED-NEXT: image_load v0, v[0:1], s[0:7] dmask:0x3 unorm d16 +; PACKED-NEXT: s_waitcnt vmcnt(0) +; PACKED-NEXT: ; return to shader part epilog +; +; GFX10-LABEL: image_load_v2f16: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: image_load v0, v[0:1], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm d16 +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: ; return to shader part epilog +; +; GFX12PLUS-LABEL: image_load_v2f16: +; GFX12PLUS: ; %bb.0: ; %main_body +; GFX12PLUS-NEXT: image_load v0, [v0, v1], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D d16 +; GFX12PLUS-NEXT: s_wait_loadcnt 0x0 +; GFX12PLUS-NEXT: ; return to shader part epilog main_body: %tex = call <2 x half> @llvm.amdgcn.image.load.2d.v2f16.i32(i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0) %r = bitcast <2 x half> %tex to float @@ -35,6 +96,38 @@ ; GFX10: image_load v[0:1], v[0:1], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_2D unorm d16{{$}} ; GFX12PLUS: image_load v[0:1], [v0, v1], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_2D d16 define amdgpu_ps <2 x float> @image_load_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) { +; UNPACKED-LABEL: image_load_v3f16: +; UNPACKED: ; %bb.0: ; %main_body +; UNPACKED-NEXT: image_load v[0:2], v[0:1], s[0:7] dmask:0x7 unorm d16 +; UNPACKED-NEXT: s_mov_b32 s0, 0x1000504 +; UNPACKED-NEXT: s_waitcnt vmcnt(0) +; UNPACKED-NEXT: v_perm_b32 v0, v0, v1, s0 +; UNPACKED-NEXT: v_mov_b32_e32 v1, v2 +; UNPACKED-NEXT: ; return to shader part epilog +; +; GFX81-LABEL: image_load_v3f16: +; GFX81: ; %bb.0: ; %main_body +; GFX81-NEXT: image_load v[0:1], v[0:1], s[0:7] dmask:0x7 unorm d16 +; GFX81-NEXT: s_waitcnt vmcnt(0) +; GFX81-NEXT: ; return to shader part epilog +; +; PACKED-LABEL: image_load_v3f16: +; PACKED: ; %bb.0: ; %main_body +; PACKED-NEXT: image_load v[0:1], v[0:1], s[0:7] dmask:0x7 unorm d16 +; PACKED-NEXT: s_waitcnt vmcnt(0) +; PACKED-NEXT: ; return to shader part epilog +; +; GFX10-LABEL: image_load_v3f16: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: image_load v[0:1], v[0:1], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_2D unorm d16 +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: ; return to shader part epilog +; +; GFX12PLUS-LABEL: image_load_v3f16: +; GFX12PLUS: ; %bb.0: ; %main_body +; GFX12PLUS-NEXT: image_load v[0:1], [v0, v1], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_2D d16 +; GFX12PLUS-NEXT: s_wait_loadcnt 0x0 +; GFX12PLUS-NEXT: ; return to shader part epilog main_body: %tex = call <3 x half> @llvm.amdgcn.image.load.2d.v3f16.i32(i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0) %ext = shufflevector <3 x half> %tex, <3 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> @@ -49,6 +142,38 @@ ; GFX10: image_load v[0:1], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm d16{{$}} ; GFX12PLUS: image_load v[0:1], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D d16 define amdgpu_ps <2 x float> @image_load_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) { +; UNPACKED-LABEL: image_load_v4f16: +; UNPACKED: ; %bb.0: ; %main_body +; UNPACKED-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm d16 +; UNPACKED-NEXT: s_mov_b32 s0, 0x1000504 +; UNPACKED-NEXT: s_waitcnt vmcnt(0) +; UNPACKED-NEXT: v_perm_b32 v0, v0, v1, s0 +; UNPACKED-NEXT: v_perm_b32 v1, v2, v3, s0 +; UNPACKED-NEXT: ; return to shader part epilog +; +; GFX81-LABEL: image_load_v4f16: +; GFX81: ; %bb.0: ; %main_body +; GFX81-NEXT: image_load v[0:1], v[0:1], s[0:7] dmask:0xf unorm d16 +; GFX81-NEXT: s_waitcnt vmcnt(0) +; GFX81-NEXT: ; return to shader part epilog +; +; PACKED-LABEL: image_load_v4f16: +; PACKED: ; %bb.0: ; %main_body +; PACKED-NEXT: image_load v[0:1], v[0:1], s[0:7] dmask:0xf unorm d16 +; PACKED-NEXT: s_waitcnt vmcnt(0) +; PACKED-NEXT: ; return to shader part epilog +; +; GFX10-LABEL: image_load_v4f16: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: image_load v[0:1], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm d16 +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: ; return to shader part epilog +; +; GFX12PLUS-LABEL: image_load_v4f16: +; GFX12PLUS: ; %bb.0: ; %main_body +; GFX12PLUS-NEXT: image_load v[0:1], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D d16 +; GFX12PLUS-NEXT: s_wait_loadcnt 0x0 +; GFX12PLUS-NEXT: ; return to shader part epilog main_body: %tex = call <4 x half> @llvm.amdgcn.image.load.2d.v4f16.i32(i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0) %r = bitcast <4 x half> %tex to <2 x float> @@ -62,6 +187,38 @@ ; GFX10: image_load_mip v[0:1], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm d16{{$}} ; GFX12PLUS: image_load_mip v[0:1], [v0, v1, v2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D d16 define amdgpu_ps <2 x float> @image_load_mip_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %mip) { +; UNPACKED-LABEL: image_load_mip_v4f16: +; UNPACKED: ; %bb.0: ; %main_body +; UNPACKED-NEXT: image_load_mip v[0:3], v[0:2], s[0:7] dmask:0xf unorm d16 +; UNPACKED-NEXT: s_mov_b32 s0, 0x1000504 +; UNPACKED-NEXT: s_waitcnt vmcnt(0) +; UNPACKED-NEXT: v_perm_b32 v0, v0, v1, s0 +; UNPACKED-NEXT: v_perm_b32 v1, v2, v3, s0 +; UNPACKED-NEXT: ; return to shader part epilog +; +; GFX81-LABEL: image_load_mip_v4f16: +; GFX81: ; %bb.0: ; %main_body +; GFX81-NEXT: image_load_mip v[0:1], v[0:2], s[0:7] dmask:0xf unorm d16 +; GFX81-NEXT: s_waitcnt vmcnt(0) +; GFX81-NEXT: ; return to shader part epilog +; +; PACKED-LABEL: image_load_mip_v4f16: +; PACKED: ; %bb.0: ; %main_body +; PACKED-NEXT: image_load_mip v[0:1], v[0:2], s[0:7] dmask:0xf unorm d16 +; PACKED-NEXT: s_waitcnt vmcnt(0) +; PACKED-NEXT: ; return to shader part epilog +; +; GFX10-LABEL: image_load_mip_v4f16: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: image_load_mip v[0:1], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm d16 +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: ; return to shader part epilog +; +; GFX12PLUS-LABEL: image_load_mip_v4f16: +; GFX12PLUS: ; %bb.0: ; %main_body +; GFX12PLUS-NEXT: image_load_mip v[0:1], [v0, v1, v2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D d16 +; GFX12PLUS-NEXT: s_wait_loadcnt 0x0 +; GFX12PLUS-NEXT: ; return to shader part epilog main_body: %tex = call <4 x half> @llvm.amdgcn.image.load.mip.2d.v4f16.i32(i32 15, i32 %s, i32 %t, i32 %mip, <8 x i32> %rsrc, i32 0, i32 0) %r = bitcast <4 x half> %tex to <2 x float> @@ -75,6 +232,37 @@ ; GFX10: image_load v0, v[0:2], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D unorm d16{{$}} ; GFX12PLUS: image_load v0, [v0, v1, v2], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D d16 define amdgpu_ps float @image_load_3d_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %r) { +; UNPACKED-LABEL: image_load_3d_v2f16: +; UNPACKED: ; %bb.0: ; %main_body +; UNPACKED-NEXT: image_load v[0:1], v[0:2], s[0:7] dmask:0x3 unorm d16 +; UNPACKED-NEXT: s_waitcnt vmcnt(0) +; UNPACKED-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; UNPACKED-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; UNPACKED-NEXT: ; return to shader part epilog +; +; GFX81-LABEL: image_load_3d_v2f16: +; GFX81: ; %bb.0: ; %main_body +; GFX81-NEXT: image_load v0, v[0:2], s[0:7] dmask:0x3 unorm d16 +; GFX81-NEXT: s_waitcnt vmcnt(0) +; GFX81-NEXT: ; return to shader part epilog +; +; PACKED-LABEL: image_load_3d_v2f16: +; PACKED: ; %bb.0: ; %main_body +; PACKED-NEXT: image_load v0, v[0:2], s[0:7] dmask:0x3 unorm d16 +; PACKED-NEXT: s_waitcnt vmcnt(0) +; PACKED-NEXT: ; return to shader part epilog +; +; GFX10-LABEL: image_load_3d_v2f16: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: image_load v0, v[0:2], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D unorm d16 +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: ; return to shader part epilog +; +; GFX12PLUS-LABEL: image_load_3d_v2f16: +; GFX12PLUS: ; %bb.0: ; %main_body +; GFX12PLUS-NEXT: image_load v0, [v0, v1, v2], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D d16 +; GFX12PLUS-NEXT: s_wait_loadcnt 0x0 +; GFX12PLUS-NEXT: ; return to shader part epilog main_body: %tex = call <2 x half> @llvm.amdgcn.image.load.3d.v2f16.i32(i32 3, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 0, i32 0) %x = bitcast <2 x half> %tex to float @@ -89,6 +277,38 @@ ; GFX10: image_load v[0:1], v[0:2], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_3D unorm d16{{$}} ; GFX12PLUS: image_load v[0:1], [v0, v1, v2], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_3D d16 define amdgpu_ps <2 x float> @image_load_3d_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %r) { +; UNPACKED-LABEL: image_load_3d_v3f16: +; UNPACKED: ; %bb.0: ; %main_body +; UNPACKED-NEXT: image_load v[0:2], v[0:2], s[0:7] dmask:0x7 unorm d16 +; UNPACKED-NEXT: s_mov_b32 s0, 0x1000504 +; UNPACKED-NEXT: s_waitcnt vmcnt(0) +; UNPACKED-NEXT: v_perm_b32 v0, v0, v1, s0 +; UNPACKED-NEXT: v_mov_b32_e32 v1, v2 +; UNPACKED-NEXT: ; return to shader part epilog +; +; GFX81-LABEL: image_load_3d_v3f16: +; GFX81: ; %bb.0: ; %main_body +; GFX81-NEXT: image_load v[0:1], v[0:2], s[0:7] dmask:0x7 unorm d16 +; GFX81-NEXT: s_waitcnt vmcnt(0) +; GFX81-NEXT: ; return to shader part epilog +; +; PACKED-LABEL: image_load_3d_v3f16: +; PACKED: ; %bb.0: ; %main_body +; PACKED-NEXT: image_load v[0:1], v[0:2], s[0:7] dmask:0x7 unorm d16 +; PACKED-NEXT: s_waitcnt vmcnt(0) +; PACKED-NEXT: ; return to shader part epilog +; +; GFX10-LABEL: image_load_3d_v3f16: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: image_load v[0:1], v[0:2], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_3D unorm d16 +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: ; return to shader part epilog +; +; GFX12PLUS-LABEL: image_load_3d_v3f16: +; GFX12PLUS: ; %bb.0: ; %main_body +; GFX12PLUS-NEXT: image_load v[0:1], [v0, v1, v2], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_3D d16 +; GFX12PLUS-NEXT: s_wait_loadcnt 0x0 +; GFX12PLUS-NEXT: ; return to shader part epilog main_body: %tex = call <3 x half> @llvm.amdgcn.image.load.3d.v3f16.i32(i32 7, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 0, i32 0) %ext = shufflevector <3 x half> %tex, <3 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> @@ -101,6 +321,30 @@ ; GFX10: image_store v2, v[0:1], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm d16{{$}} ; GFX12PLUS: image_store v2, [v0, v1], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D d16 define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, half %data) { +; UNPACKED-LABEL: image_store_f16: +; UNPACKED: ; %bb.0: ; %main_body +; UNPACKED-NEXT: image_store v2, v[0:1], s[0:7] dmask:0x1 unorm d16 +; UNPACKED-NEXT: s_endpgm +; +; GFX81-LABEL: image_store_f16: +; GFX81: ; %bb.0: ; %main_body +; GFX81-NEXT: image_store v2, v[0:1], s[0:7] dmask:0x1 unorm d16 +; GFX81-NEXT: s_endpgm +; +; PACKED-LABEL: image_store_f16: +; PACKED: ; %bb.0: ; %main_body +; PACKED-NEXT: image_store v2, v[0:1], s[0:7] dmask:0x1 unorm d16 +; PACKED-NEXT: s_endpgm +; +; GFX10-LABEL: image_store_f16: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: image_store v2, v[0:1], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm d16 +; GFX10-NEXT: s_endpgm +; +; GFX12PLUS-LABEL: image_store_f16: +; GFX12PLUS: ; %bb.0: ; %main_body +; GFX12PLUS-NEXT: image_store v2, [v0, v1], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D d16 +; GFX12PLUS-NEXT: s_endpgm main_body: call void @llvm.amdgcn.image.store.2d.f16.i32(half %data, i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0) ret void @@ -115,6 +359,32 @@ ; GFX10: image_store v2, v[0:1], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm d16{{$}} ; GFX12PLUS: image_store v2, [v0, v1], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D d16 define amdgpu_ps void @image_store_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, float %in) { +; UNPACKED-LABEL: image_store_v2f16: +; UNPACKED: ; %bb.0: ; %main_body +; UNPACKED-NEXT: v_lshrrev_b32_e32 v3, 16, v2 +; UNPACKED-NEXT: v_and_b32_e32 v2, 0xffff, v2 +; UNPACKED-NEXT: image_store v[2:3], v[0:1], s[0:7] dmask:0x3 unorm d16 +; UNPACKED-NEXT: s_endpgm +; +; GFX81-LABEL: image_store_v2f16: +; GFX81: ; %bb.0: ; %main_body +; GFX81-NEXT: image_store v[2:3], v[0:1], s[0:7] dmask:0x3 unorm d16 +; GFX81-NEXT: s_endpgm +; +; PACKED-LABEL: image_store_v2f16: +; PACKED: ; %bb.0: ; %main_body +; PACKED-NEXT: image_store v2, v[0:1], s[0:7] dmask:0x3 unorm d16 +; PACKED-NEXT: s_endpgm +; +; GFX10-LABEL: image_store_v2f16: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: image_store v2, v[0:1], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm d16 +; GFX10-NEXT: s_endpgm +; +; GFX12PLUS-LABEL: image_store_v2f16: +; GFX12PLUS: ; %bb.0: ; %main_body +; GFX12PLUS-NEXT: image_store v2, [v0, v1], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D d16 +; GFX12PLUS-NEXT: s_endpgm main_body: %data = bitcast float %in to <2 x half> call void @llvm.amdgcn.image.store.2d.v2f16.i32(<2 x half> %data, i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0) @@ -128,6 +398,36 @@ ; GFX10: image_store v[2:3], v[0:1], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_2D unorm d16{{$}} ; GFX12PLUS: image_store v[2:3], [v0, v1], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_2D d16 define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <2 x float> %in) { +; UNPACKED-LABEL: image_store_v3f16: +; UNPACKED: ; %bb.0: ; %main_body +; UNPACKED-NEXT: v_and_b32_e32 v4, 0xffff, v3 +; UNPACKED-NEXT: v_lshrrev_b32_e32 v3, 16, v2 +; UNPACKED-NEXT: v_and_b32_e32 v2, 0xffff, v2 +; UNPACKED-NEXT: image_store v[2:4], v[0:1], s[0:7] dmask:0x7 unorm d16 +; UNPACKED-NEXT: s_endpgm +; +; GFX81-LABEL: image_store_v3f16: +; GFX81: ; %bb.0: ; %main_body +; GFX81-NEXT: image_store v[2:4], v[0:1], s[0:7] dmask:0x7 unorm d16 +; GFX81-NEXT: s_endpgm +; +; PACKED-LABEL: image_store_v3f16: +; PACKED: ; %bb.0: ; %main_body +; PACKED-NEXT: v_and_b32_e32 v3, 0xffff, v3 +; PACKED-NEXT: image_store v[2:3], v[0:1], s[0:7] dmask:0x7 unorm d16 +; PACKED-NEXT: s_endpgm +; +; GFX10-LABEL: image_store_v3f16: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3 +; GFX10-NEXT: image_store v[2:3], v[0:1], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_2D unorm d16 +; GFX10-NEXT: s_endpgm +; +; GFX12PLUS-LABEL: image_store_v3f16: +; GFX12PLUS: ; %bb.0: ; %main_body +; GFX12PLUS-NEXT: v_and_b32_e32 v3, 0xffff, v3 +; GFX12PLUS-NEXT: image_store v[2:3], [v0, v1], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_2D d16 +; GFX12PLUS-NEXT: s_endpgm main_body: %r = bitcast <2 x float> %in to <4 x half> %data = shufflevector <4 x half> %r, <4 x half> poison, <3 x i32> <i32 0, i32 1, i32 2> @@ -146,6 +446,34 @@ ; GFX10: image_store v[2:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm d16{{$}} ; GFX12PLUS: image_store v[2:3], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D d16 define amdgpu_ps void @image_store_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <2 x float> %in) { +; UNPACKED-LABEL: image_store_v4f16: +; UNPACKED: ; %bb.0: ; %main_body +; UNPACKED-NEXT: v_lshrrev_b32_e32 v5, 16, v3 +; UNPACKED-NEXT: v_and_b32_e32 v4, 0xffff, v3 +; UNPACKED-NEXT: v_lshrrev_b32_e32 v3, 16, v2 +; UNPACKED-NEXT: v_and_b32_e32 v2, 0xffff, v2 +; UNPACKED-NEXT: image_store v[2:5], v[0:1], s[0:7] dmask:0xf unorm d16 +; UNPACKED-NEXT: s_endpgm +; +; GFX81-LABEL: image_store_v4f16: +; GFX81: ; %bb.0: ; %main_body +; GFX81-NEXT: image_store v[2:5], v[0:1], s[0:7] dmask:0xf unorm d16 +; GFX81-NEXT: s_endpgm +; +; PACKED-LABEL: image_store_v4f16: +; PACKED: ; %bb.0: ; %main_body +; PACKED-NEXT: image_store v[2:3], v[0:1], s[0:7] dmask:0xf unorm d16 +; PACKED-NEXT: s_endpgm +; +; GFX10-LABEL: image_store_v4f16: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: image_store v[2:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm d16 +; GFX10-NEXT: s_endpgm +; +; GFX12PLUS-LABEL: image_store_v4f16: +; GFX12PLUS: ; %bb.0: ; %main_body +; GFX12PLUS-NEXT: image_store v[2:3], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D d16 +; GFX12PLUS-NEXT: s_endpgm main_body: %data = bitcast <2 x float> %in to <4 x half> call void @llvm.amdgcn.image.store.2d.v4f16.i32(<4 x half> %data, i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0) @@ -163,6 +491,34 @@ ; GFX10: image_store_mip v[2:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm d16{{$}} ; GFX12PLUS: image_store_mip v[2:3], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D d16 define amdgpu_ps void @image_store_mip_1d_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %mip, <2 x float> %in) { +; UNPACKED-LABEL: image_store_mip_1d_v4f16: +; UNPACKED: ; %bb.0: ; %main_body +; UNPACKED-NEXT: v_lshrrev_b32_e32 v5, 16, v3 +; UNPACKED-NEXT: v_and_b32_e32 v4, 0xffff, v3 +; UNPACKED-NEXT: v_lshrrev_b32_e32 v3, 16, v2 +; UNPACKED-NEXT: v_and_b32_e32 v2, 0xffff, v2 +; UNPACKED-NEXT: image_store_mip v[2:5], v[0:1], s[0:7] dmask:0xf unorm d16 +; UNPACKED-NEXT: s_endpgm +; +; GFX81-LABEL: image_store_mip_1d_v4f16: +; GFX81: ; %bb.0: ; %main_body +; GFX81-NEXT: image_store_mip v[2:5], v[0:1], s[0:7] dmask:0xf unorm d16 +; GFX81-NEXT: s_endpgm +; +; PACKED-LABEL: image_store_mip_1d_v4f16: +; PACKED: ; %bb.0: ; %main_body +; PACKED-NEXT: image_store_mip v[2:3], v[0:1], s[0:7] dmask:0xf unorm d16 +; PACKED-NEXT: s_endpgm +; +; GFX10-LABEL: image_store_mip_1d_v4f16: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: image_store_mip v[2:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm d16 +; GFX10-NEXT: s_endpgm +; +; GFX12PLUS-LABEL: image_store_mip_1d_v4f16: +; GFX12PLUS: ; %bb.0: ; %main_body +; GFX12PLUS-NEXT: image_store_mip v[2:3], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D d16 +; GFX12PLUS-NEXT: s_endpgm main_body: %data = bitcast <2 x float> %in to <4 x half> call void @llvm.amdgcn.image.store.mip.1d.v4f16.i32(<4 x half> %data, i32 15, i32 %s, i32 %mip, <8 x i32> %rsrc, i32 0, i32 0) @@ -188,3 +544,6 @@ attributes #0 = { nounwind } attributes #1 = { nounwind readonly } attributes #2 = { nounwind readnone } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GCN: {{.*}} +; GFX89: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll index c2b6224..c5e03d2 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll
@@ -35,15 +35,25 @@ ; GFX10-PACKED-NEXT: tbuffer_store_format_d16_x v0, off, s[0:3], 0 format:[BUF_FMT_10_11_11_SSCALED] ; GFX10-PACKED-NEXT: s_endpgm ; -; GFX11-PACKED-LABEL: tbuffer_store_d16_x: -; GFX11-PACKED: ; %bb.0: ; %main_body -; GFX11-PACKED-NEXT: s_clause 0x1 -; GFX11-PACKED-NEXT: s_load_b32 s6, s[4:5], 0x34 -; GFX11-PACKED-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 -; GFX11-PACKED-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, s6 -; GFX11-PACKED-NEXT: tbuffer_store_d16_format_x v0, off, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] -; GFX11-PACKED-NEXT: s_endpgm +; GFX11-PACKED-TRUE16-LABEL: tbuffer_store_d16_x: +; GFX11-PACKED-TRUE16: ; %bb.0: ; %main_body +; GFX11-PACKED-TRUE16-NEXT: s_clause 0x1 +; GFX11-PACKED-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 +; GFX11-PACKED-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 +; GFX11-PACKED-TRUE16-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PACKED-TRUE16-NEXT: v_mov_b16_e32 v0.l, s6 +; GFX11-PACKED-TRUE16-NEXT: tbuffer_store_d16_format_x v0, off, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] +; GFX11-PACKED-TRUE16-NEXT: s_endpgm +; +; GFX11-PACKED-FAKE16-LABEL: tbuffer_store_d16_x: +; GFX11-PACKED-FAKE16: ; %bb.0: ; %main_body +; GFX11-PACKED-FAKE16-NEXT: s_clause 0x1 +; GFX11-PACKED-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 +; GFX11-PACKED-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 +; GFX11-PACKED-FAKE16-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PACKED-FAKE16-NEXT: v_mov_b32_e32 v0, s6 +; GFX11-PACKED-FAKE16-NEXT: tbuffer_store_d16_format_x v0, off, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] +; GFX11-PACKED-FAKE16-NEXT: s_endpgm main_body: call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %data, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 33, i32 0) ret void @@ -325,6 +335,3 @@ declare void @llvm.amdgcn.raw.ptr.tbuffer.store.bf16(bfloat, ptr addrspace(8), i32, i32, i32, i32) declare void @llvm.amdgcn.raw.ptr.tbuffer.store.v2bf16(<2 x bfloat>, ptr addrspace(8), i32, i32, i32, i32) declare void @llvm.amdgcn.raw.ptr.tbuffer.store.v4bf16(<4 x bfloat>, ptr addrspace(8), i32, i32, i32, i32) -;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; GFX11-PACKED-FAKE16: {{.*}} -; GFX11-PACKED-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll index c1a92f8..7e19765 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll
@@ -39,25 +39,55 @@ ; GFX10-PACKED-NEXT: tbuffer_store_format_d16_x v0, off, s[0:3], 0 format:[BUF_FMT_10_11_11_SSCALED] ; GFX10-PACKED-NEXT: s_endpgm ; -; GFX11-PACKED-LABEL: tbuffer_store_d16_x: -; GFX11-PACKED: ; %bb.0: ; %main_body -; GFX11-PACKED-NEXT: s_clause 0x1 -; GFX11-PACKED-NEXT: s_load_b32 s6, s[4:5], 0x34 -; GFX11-PACKED-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 -; GFX11-PACKED-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, s6 -; GFX11-PACKED-NEXT: tbuffer_store_d16_format_x v0, off, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] -; GFX11-PACKED-NEXT: s_endpgm +; GFX11-PACKED-TRUE16-LABEL: tbuffer_store_d16_x: +; GFX11-PACKED-TRUE16: ; %bb.0: ; %main_body +; GFX11-PACKED-TRUE16-NEXT: s_clause 0x1 +; GFX11-PACKED-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 +; GFX11-PACKED-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 +; GFX11-PACKED-TRUE16-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PACKED-TRUE16-NEXT: v_mov_b16_e32 v0.l, s6 +; GFX11-PACKED-TRUE16-NEXT: tbuffer_store_d16_format_x v0, off, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] +; GFX11-PACKED-TRUE16-NEXT: s_endpgm ; -; GFX12-PACKED-LABEL: tbuffer_store_d16_x: -; GFX12-PACKED: ; %bb.0: ; %main_body -; GFX12-PACKED-NEXT: s_clause 0x1 -; GFX12-PACKED-NEXT: s_load_b32 s6, s[4:5], 0x34 -; GFX12-PACKED-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 -; GFX12-PACKED-NEXT: s_wait_kmcnt 0x0 -; GFX12-PACKED-NEXT: v_mov_b32_e32 v0, s6 -; GFX12-PACKED-NEXT: tbuffer_store_d16_format_x v0, off, s[0:3], null format:[BUF_FMT_10_10_10_2_SNORM] -; GFX12-PACKED-NEXT: s_endpgm +; GFX11-PACKED-FAKE16-LABEL: tbuffer_store_d16_x: +; GFX11-PACKED-FAKE16: ; %bb.0: ; %main_body +; GFX11-PACKED-FAKE16-NEXT: s_clause 0x1 +; GFX11-PACKED-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 +; GFX11-PACKED-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 +; GFX11-PACKED-FAKE16-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PACKED-FAKE16-NEXT: v_mov_b32_e32 v0, s6 +; GFX11-PACKED-FAKE16-NEXT: tbuffer_store_d16_format_x v0, off, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] +; GFX11-PACKED-FAKE16-NEXT: s_endpgm +; +; GFX12-PACKED-SDAG-TRUE16-LABEL: tbuffer_store_d16_x: +; GFX12-PACKED-SDAG-TRUE16: ; %bb.0: ; %main_body +; GFX12-PACKED-SDAG-TRUE16-NEXT: s_clause 0x1 +; GFX12-PACKED-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 +; GFX12-PACKED-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 +; GFX12-PACKED-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-PACKED-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, s6 +; GFX12-PACKED-SDAG-TRUE16-NEXT: tbuffer_store_d16_format_x v0, off, s[0:3], null format:[BUF_FMT_10_10_10_2_SNORM] +; GFX12-PACKED-SDAG-TRUE16-NEXT: s_endpgm +; +; GFX12-PACKED-SDAG-FAKE16-LABEL: tbuffer_store_d16_x: +; GFX12-PACKED-SDAG-FAKE16: ; %bb.0: ; %main_body +; GFX12-PACKED-SDAG-FAKE16-NEXT: s_clause 0x1 +; GFX12-PACKED-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 +; GFX12-PACKED-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 +; GFX12-PACKED-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-PACKED-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s6 +; GFX12-PACKED-SDAG-FAKE16-NEXT: tbuffer_store_d16_format_x v0, off, s[0:3], null format:[BUF_FMT_10_10_10_2_SNORM] +; GFX12-PACKED-SDAG-FAKE16-NEXT: s_endpgm +; +; GFX12-PACKED-GISEL-LABEL: tbuffer_store_d16_x: +; GFX12-PACKED-GISEL: ; %bb.0: ; %main_body +; GFX12-PACKED-GISEL-NEXT: s_clause 0x1 +; GFX12-PACKED-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34 +; GFX12-PACKED-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 +; GFX12-PACKED-GISEL-NEXT: s_wait_kmcnt 0x0 +; GFX12-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, s6 +; GFX12-PACKED-GISEL-NEXT: tbuffer_store_d16_format_x v0, off, s[0:3], null format:[BUF_FMT_10_10_10_2_SNORM] +; GFX12-PACKED-GISEL-NEXT: s_endpgm main_body: call void @llvm.amdgcn.raw.tbuffer.store.f16(half %data, <4 x i32> %rsrc, i32 0, i32 0, i32 33, i32 0) ret void @@ -266,9 +296,5 @@ declare void @llvm.amdgcn.raw.tbuffer.store.v3f16(<3 x half>, <4 x i32>, i32, i32, i32, i32) declare void @llvm.amdgcn.raw.tbuffer.store.v4f16(<4 x half>, <4 x i32>, i32, i32, i32, i32) ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; GFX11-PACKED-FAKE16: {{.*}} -; GFX11-PACKED-TRUE16: {{.*}} ; GFX12-PACKED-GISEL-FAKE16: {{.*}} ; GFX12-PACKED-GISEL-TRUE16: {{.*}} -; GFX12-PACKED-SDAG-FAKE16: {{.*}} -; GFX12-PACKED-SDAG-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.store.d16.ll index 7a9cc7c..ede921a 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.store.d16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.store.d16.ll
@@ -38,16 +38,27 @@ ; GFX10-PACKED-NEXT: tbuffer_store_format_d16_x v0, v1, s[0:3], 0 format:[BUF_FMT_10_11_11_SSCALED] idxen ; GFX10-PACKED-NEXT: s_endpgm ; -; GFX11-PACKED-LABEL: tbuffer_store_d16_x: -; GFX11-PACKED: ; %bb.0: ; %main_body -; GFX11-PACKED-NEXT: s_clause 0x1 -; GFX11-PACKED-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 -; GFX11-PACKED-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 -; GFX11-PACKED-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, s6 -; GFX11-PACKED-NEXT: v_mov_b32_e32 v1, s7 -; GFX11-PACKED-NEXT: tbuffer_store_d16_format_x v0, v1, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] idxen -; GFX11-PACKED-NEXT: s_endpgm +; GFX11-PACKED-TRUE16-LABEL: tbuffer_store_d16_x: +; GFX11-PACKED-TRUE16: ; %bb.0: ; %main_body +; GFX11-PACKED-TRUE16-NEXT: s_clause 0x1 +; GFX11-PACKED-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 +; GFX11-PACKED-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 +; GFX11-PACKED-TRUE16-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PACKED-TRUE16-NEXT: v_mov_b16_e32 v0.l, s6 +; GFX11-PACKED-TRUE16-NEXT: v_mov_b32_e32 v1, s7 +; GFX11-PACKED-TRUE16-NEXT: tbuffer_store_d16_format_x v0, v1, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] idxen +; GFX11-PACKED-TRUE16-NEXT: s_endpgm +; +; GFX11-PACKED-FAKE16-LABEL: tbuffer_store_d16_x: +; GFX11-PACKED-FAKE16: ; %bb.0: ; %main_body +; GFX11-PACKED-FAKE16-NEXT: s_clause 0x1 +; GFX11-PACKED-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 +; GFX11-PACKED-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 +; GFX11-PACKED-FAKE16-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PACKED-FAKE16-NEXT: v_mov_b32_e32 v0, s6 +; GFX11-PACKED-FAKE16-NEXT: v_mov_b32_e32 v1, s7 +; GFX11-PACKED-FAKE16-NEXT: tbuffer_store_d16_format_x v0, v1, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] idxen +; GFX11-PACKED-FAKE16-NEXT: s_endpgm main_body: call void @llvm.amdgcn.struct.ptr.tbuffer.store.f16(half %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0, i32 33, i32 0) ret void @@ -231,6 +242,3 @@ declare void @llvm.amdgcn.struct.ptr.tbuffer.store.v2f16(<2 x half>, ptr addrspace(8), i32, i32, i32, i32, i32) declare void @llvm.amdgcn.struct.ptr.tbuffer.store.v3f16(<3 x half>, ptr addrspace(8), i32, i32, i32, i32, i32) declare void @llvm.amdgcn.struct.ptr.tbuffer.store.v4f16(<4 x half>, ptr addrspace(8), i32, i32, i32, i32, i32) -;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; GFX11-PACKED-FAKE16: {{.*}} -; GFX11-PACKED-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll index 94668c0..f9c7eb7 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll
@@ -42,27 +42,60 @@ ; GFX10-PACKED-NEXT: tbuffer_store_format_d16_x v0, v1, s[0:3], 0 format:[BUF_FMT_10_11_11_SSCALED] idxen ; GFX10-PACKED-NEXT: s_endpgm ; -; GFX11-PACKED-LABEL: tbuffer_store_d16_x: -; GFX11-PACKED: ; %bb.0: ; %main_body -; GFX11-PACKED-NEXT: s_clause 0x1 -; GFX11-PACKED-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 -; GFX11-PACKED-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 -; GFX11-PACKED-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, s6 -; GFX11-PACKED-NEXT: v_mov_b32_e32 v1, s7 -; GFX11-PACKED-NEXT: tbuffer_store_d16_format_x v0, v1, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] idxen -; GFX11-PACKED-NEXT: s_endpgm +; GFX11-PACKED-TRUE16-LABEL: tbuffer_store_d16_x: +; GFX11-PACKED-TRUE16: ; %bb.0: ; %main_body +; GFX11-PACKED-TRUE16-NEXT: s_clause 0x1 +; GFX11-PACKED-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 +; GFX11-PACKED-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 +; GFX11-PACKED-TRUE16-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PACKED-TRUE16-NEXT: v_mov_b16_e32 v0.l, s6 +; GFX11-PACKED-TRUE16-NEXT: v_mov_b32_e32 v1, s7 +; GFX11-PACKED-TRUE16-NEXT: tbuffer_store_d16_format_x v0, v1, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] idxen +; GFX11-PACKED-TRUE16-NEXT: s_endpgm ; -; GFX12-PACKED-LABEL: tbuffer_store_d16_x: -; GFX12-PACKED: ; %bb.0: ; %main_body -; GFX12-PACKED-NEXT: s_clause 0x1 -; GFX12-PACKED-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 -; GFX12-PACKED-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 -; GFX12-PACKED-NEXT: s_wait_kmcnt 0x0 -; GFX12-PACKED-NEXT: v_mov_b32_e32 v0, s6 -; GFX12-PACKED-NEXT: v_mov_b32_e32 v1, s7 -; GFX12-PACKED-NEXT: tbuffer_store_d16_format_x v0, v1, s[0:3], null format:[BUF_FMT_10_10_10_2_SNORM] idxen -; GFX12-PACKED-NEXT: s_endpgm +; GFX11-PACKED-FAKE16-LABEL: tbuffer_store_d16_x: +; GFX11-PACKED-FAKE16: ; %bb.0: ; %main_body +; GFX11-PACKED-FAKE16-NEXT: s_clause 0x1 +; GFX11-PACKED-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 +; GFX11-PACKED-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 +; GFX11-PACKED-FAKE16-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PACKED-FAKE16-NEXT: v_mov_b32_e32 v0, s6 +; GFX11-PACKED-FAKE16-NEXT: v_mov_b32_e32 v1, s7 +; GFX11-PACKED-FAKE16-NEXT: tbuffer_store_d16_format_x v0, v1, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] idxen +; GFX11-PACKED-FAKE16-NEXT: s_endpgm +; +; GFX12-PACKED-SDAG-TRUE16-LABEL: tbuffer_store_d16_x: +; GFX12-PACKED-SDAG-TRUE16: ; %bb.0: ; %main_body +; GFX12-PACKED-SDAG-TRUE16-NEXT: s_clause 0x1 +; GFX12-PACKED-SDAG-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 +; GFX12-PACKED-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 +; GFX12-PACKED-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-PACKED-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, s6 +; GFX12-PACKED-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s7 +; GFX12-PACKED-SDAG-TRUE16-NEXT: tbuffer_store_d16_format_x v0, v1, s[0:3], null format:[BUF_FMT_10_10_10_2_SNORM] idxen +; GFX12-PACKED-SDAG-TRUE16-NEXT: s_endpgm +; +; GFX12-PACKED-SDAG-FAKE16-LABEL: tbuffer_store_d16_x: +; GFX12-PACKED-SDAG-FAKE16: ; %bb.0: ; %main_body +; GFX12-PACKED-SDAG-FAKE16-NEXT: s_clause 0x1 +; GFX12-PACKED-SDAG-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 +; GFX12-PACKED-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 +; GFX12-PACKED-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-PACKED-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s6 +; GFX12-PACKED-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s7 +; GFX12-PACKED-SDAG-FAKE16-NEXT: tbuffer_store_d16_format_x v0, v1, s[0:3], null format:[BUF_FMT_10_10_10_2_SNORM] idxen +; GFX12-PACKED-SDAG-FAKE16-NEXT: s_endpgm +; +; GFX12-PACKED-GISEL-LABEL: tbuffer_store_d16_x: +; GFX12-PACKED-GISEL: ; %bb.0: ; %main_body +; GFX12-PACKED-GISEL-NEXT: s_clause 0x1 +; GFX12-PACKED-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 +; GFX12-PACKED-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 +; GFX12-PACKED-GISEL-NEXT: s_wait_kmcnt 0x0 +; GFX12-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, s6 +; GFX12-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, s7 +; GFX12-PACKED-GISEL-NEXT: tbuffer_store_d16_format_x v0, v1, s[0:3], null format:[BUF_FMT_10_10_10_2_SNORM] idxen +; GFX12-PACKED-GISEL-NEXT: s_endpgm main_body: call void @llvm.amdgcn.struct.tbuffer.store.f16(half %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 33, i32 0) ret void @@ -295,9 +328,5 @@ declare void @llvm.amdgcn.struct.tbuffer.store.v3f16(<3 x half>, <4 x i32>, i32, i32, i32, i32, i32) declare void @llvm.amdgcn.struct.tbuffer.store.v4f16(<4 x half>, <4 x i32>, i32, i32, i32, i32, i32) ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; GFX11-PACKED-FAKE16: {{.*}} -; GFX11-PACKED-TRUE16: {{.*}} ; GFX12-PACKED-GISEL-FAKE16: {{.*}} ; GFX12-PACKED-GISEL-TRUE16: {{.*}} -; GFX12-PACKED-SDAG-FAKE16: {{.*}} -; GFX12-PACKED-SDAG-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll index 8bde7ed..b8e2fdb 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll
@@ -17,13 +17,13 @@ ; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) ; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0 ; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1 ; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo -; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2 -; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) ; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0 ; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1 ; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) @@ -565,13 +565,13 @@ ; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) ; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0 ; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1 ; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo -; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2 -; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) ; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0 ; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1 ; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll index 60dba22..52ca7f0 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll
@@ -1039,9 +1039,9 @@ ; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0 ; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0 ; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v0.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v1.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo +; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 0x7f80, v1.l +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0 ; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11SELDAG-FAKE16-LABEL: isnan_v2bf16: @@ -1124,13 +1124,11 @@ ; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1 ; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0 ; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v0.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v2.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v1.l -; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l +; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e64 s1, 0x7f80, v1.l +; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 0x7f80, v2.l +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, 0, 1, s1 +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0 ; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11SELDAG-FAKE16-LABEL: isnan_v3bf16: @@ -1224,19 +1222,16 @@ ; GFX11SELDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0 ; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1 +; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0 +; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1 ; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v0.l -; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1 -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v1.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v3.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v2.l -; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l -; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l +; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 0x7f80, v1.l +; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e64 s1, 0x7f80, v2.l +; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e64 s2, 0x7f80, v3.l +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, 0, 1, s0 +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s1 +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, 0, 1, s2 ; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11SELDAG-FAKE16-LABEL: isnan_v4bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll index d6c5c76..6c2d6b8 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
@@ -1411,9 +1411,9 @@ ; GFX11SELDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0 ; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.l, v0.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1.l, v1.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo +; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e64 s0, v1.l, v1.l +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0 ; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11SELDAG-FAKE16-LABEL: isnan_v2f16: @@ -1564,13 +1564,11 @@ ; GFX11SELDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0 ; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.l, v0.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v2.l, v2.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1.l, v1.l -; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l +; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e64 s1, v1.l, v1.l +; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e64 s0, v2.l, v2.l +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, 0, 1, s1 +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0 ; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11SELDAG-FAKE16-LABEL: isnan_v3f16: @@ -1749,19 +1747,16 @@ ; GFX11SELDAG-TRUE16-LABEL: isnan_v4f16: ; GFX11SELDAG-TRUE16: ; %bb.0: ; GFX11SELDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0 +; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1 ; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.l, v0.l -; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1 -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1.l, v1.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v3.l, v3.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v2.l, v2.l -; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l -; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l -; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo -; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l +; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e64 s0, v1.l, v1.l +; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e64 s1, v2.l, v2.l +; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e64 s2, v3.l, v3.l +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, 0, 1, s0 +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s1 +; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, 0, 1, s2 ; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11SELDAG-FAKE16-LABEL: isnan_v4f16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll index 6398e85..2e09efe 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll
@@ -366,25 +366,15 @@ ; GFX10CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s4 ; GFX10CHECK-NEXT: s_setpc_b64 s[30:31] ; -; GFX11SELDAG-LABEL: isnan_v2f32: -; GFX11SELDAG: ; %bb.0: -; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0 -; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31] -; -; GFX11GLISEL-LABEL: isnan_v2f32: -; GFX11GLISEL: ; %bb.0: -; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3 -; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3 -; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0 -; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1 -; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31] +; GFX11CHECK-LABEL: isnan_v2f32: +; GFX11CHECK: ; %bb.0: +; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3 +; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s1, v1, 3 +; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11CHECK-NEXT: v_cndmask_b16 v0.l, 0, 1, s0 +; GFX11CHECK-NEXT: v_cndmask_b16 v1.l, 0, 1, s1 +; GFX11CHECK-NEXT: s_setpc_b64 s[30:31] %1 = call <2 x i1> @llvm.is.fpclass.v2f32(<2 x float> %x, i32 3) ; nan ret <2 x i1> %1 } @@ -434,31 +424,18 @@ ; GFX10CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s4 ; GFX10CHECK-NEXT: s_setpc_b64 s[30:31] ; -; GFX11SELDAG-LABEL: isnan_v3f32: -; GFX11SELDAG: ; %bb.0: -; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 -; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31] -; -; GFX11GLISEL-LABEL: isnan_v3f32: -; GFX11GLISEL: ; %bb.0: -; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3 -; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3 -; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3 -; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0 -; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1 -; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2 -; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31] +; GFX11CHECK-LABEL: isnan_v3f32: +; GFX11CHECK: ; %bb.0: +; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3 +; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s1, v1, 3 +; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s2, v2, 3 +; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11CHECK-NEXT: v_cndmask_b16 v0.l, 0, 1, s0 +; GFX11CHECK-NEXT: v_cndmask_b16 v1.l, 0, 1, s1 +; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11CHECK-NEXT: v_cndmask_b16 v2.l, 0, 1, s2 +; GFX11CHECK-NEXT: s_setpc_b64 s[30:31] %1 = call <3 x i1> @llvm.is.fpclass.v3f32(<3 x float> %x, i32 3) ; nan ret <3 x i1> %1 } @@ -516,35 +493,20 @@ ; GFX10CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s4 ; GFX10CHECK-NEXT: s_setpc_b64 s[30:31] ; -; GFX11SELDAG-LABEL: isnan_v4f32: -; GFX11SELDAG: ; %bb.0: -; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0 -; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31] -; -; GFX11GLISEL-LABEL: isnan_v4f32: -; GFX11GLISEL: ; %bb.0: -; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3 -; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3 -; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3 -; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s3, v3, 3 -; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0 -; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1 -; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2 -; GFX11GLISEL-NEXT: v_cndmask_b16 v3.l, 0, 1, s3 -; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31] +; GFX11CHECK-LABEL: isnan_v4f32: +; GFX11CHECK: ; %bb.0: +; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3 +; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s1, v1, 3 +; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s2, v2, 3 +; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s3, v3, 3 +; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11CHECK-NEXT: v_cndmask_b16 v0.l, 0, 1, s0 +; GFX11CHECK-NEXT: v_cndmask_b16 v1.l, 0, 1, s1 +; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11CHECK-NEXT: v_cndmask_b16 v2.l, 0, 1, s2 +; GFX11CHECK-NEXT: v_cndmask_b16 v3.l, 0, 1, s3 +; GFX11CHECK-NEXT: s_setpc_b64 s[30:31] %1 = call <4 x i1> @llvm.is.fpclass.v4f32(<4 x float> %x, i32 3) ; nan ret <4 x i1> %1 } @@ -610,38 +572,20 @@ ; GFX10CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, s4 ; GFX10CHECK-NEXT: s_setpc_b64 s[30:31] ; -; GFX11SELDAG-LABEL: isnan_v5f32: -; GFX11SELDAG: ; %bb.0: -; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v4, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0 -; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31] -; -; GFX11GLISEL-LABEL: isnan_v5f32: -; GFX11GLISEL: ; %bb.0: -; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3 -; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3 -; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3 -; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s3, v3, 3 -; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s4, v4, 3 -; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0 -; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1 -; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2 -; GFX11GLISEL-NEXT: v_cndmask_b16 v3.l, 0, 1, s3 -; GFX11GLISEL-NEXT: v_cndmask_b16 v4.l, 0, 1, s4 -; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31] +; GFX11CHECK-LABEL: isnan_v5f32: +; GFX11CHECK: ; %bb.0: +; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3 +; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s1, v1, 3 +; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s2, v2, 3 +; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s3, v3, 3 +; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s4, v4, 3 +; GFX11CHECK-NEXT: v_cndmask_b16 v0.l, 0, 1, s0 +; GFX11CHECK-NEXT: v_cndmask_b16 v1.l, 0, 1, s1 +; GFX11CHECK-NEXT: v_cndmask_b16 v2.l, 0, 1, s2 +; GFX11CHECK-NEXT: v_cndmask_b16 v3.l, 0, 1, s3 +; GFX11CHECK-NEXT: v_cndmask_b16 v4.l, 0, 1, s4 +; GFX11CHECK-NEXT: s_setpc_b64 s[30:31] %1 = call <5 x i1> @llvm.is.fpclass.v5f32(<5 x float> %x, i32 3) ; nan ret <5 x i1> %1 } @@ -718,21 +662,18 @@ ; GFX11SELDAG-LABEL: isnan_v6f32: ; GFX11SELDAG: ; %bb.0: ; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v4, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0 ; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v5, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s1, v0, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s2, v1, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s3, v2, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s4, v3, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s5, v4, 3 +; GFX11SELDAG-NEXT: v_cndmask_b16 v0.l, 0, 1, s1 +; GFX11SELDAG-NEXT: v_cndmask_b16 v1.l, 0, 1, s2 +; GFX11SELDAG-NEXT: v_cndmask_b16 v2.l, 0, 1, s3 +; GFX11SELDAG-NEXT: v_cndmask_b16 v3.l, 0, 1, s4 +; GFX11SELDAG-NEXT: v_cndmask_b16 v4.l, 0, 1, s5 +; GFX11SELDAG-NEXT: v_cndmask_b16 v5.l, 0, 1, s0 ; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31] ; ; GFX11GLISEL-LABEL: isnan_v6f32: @@ -835,24 +776,20 @@ ; GFX11SELDAG-LABEL: isnan_v7f32: ; GFX11SELDAG: ; %bb.0: ; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v4, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v5, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0 ; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v6, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s1, v5, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s2, v0, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s3, v1, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s4, v2, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s5, v3, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s6, v4, 3 +; GFX11SELDAG-NEXT: v_cndmask_b16 v0.l, 0, 1, s2 +; GFX11SELDAG-NEXT: v_cndmask_b16 v1.l, 0, 1, s3 +; GFX11SELDAG-NEXT: v_cndmask_b16 v2.l, 0, 1, s4 +; GFX11SELDAG-NEXT: v_cndmask_b16 v3.l, 0, 1, s5 +; GFX11SELDAG-NEXT: v_cndmask_b16 v4.l, 0, 1, s6 +; GFX11SELDAG-NEXT: v_cndmask_b16 v5.l, 0, 1, s1 +; GFX11SELDAG-NEXT: v_cndmask_b16 v6.l, 0, 1, s0 ; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31] ; ; GFX11GLISEL-LABEL: isnan_v7f32: @@ -965,26 +902,22 @@ ; GFX11SELDAG-LABEL: isnan_v8f32: ; GFX11SELDAG: ; %bb.0: ; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v4, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v5, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v6, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0 ; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v7, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s1, v6, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s2, v5, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s3, v0, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s4, v1, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s5, v2, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s6, v3, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s7, v4, 3 +; GFX11SELDAG-NEXT: v_cndmask_b16 v0.l, 0, 1, s3 +; GFX11SELDAG-NEXT: v_cndmask_b16 v1.l, 0, 1, s4 +; GFX11SELDAG-NEXT: v_cndmask_b16 v2.l, 0, 1, s5 +; GFX11SELDAG-NEXT: v_cndmask_b16 v3.l, 0, 1, s6 +; GFX11SELDAG-NEXT: v_cndmask_b16 v4.l, 0, 1, s7 +; GFX11SELDAG-NEXT: v_cndmask_b16 v5.l, 0, 1, s2 +; GFX11SELDAG-NEXT: v_cndmask_b16 v6.l, 0, 1, s1 +; GFX11SELDAG-NEXT: v_cndmask_b16 v7.l, 0, 1, s0 ; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31] ; ; GFX11GLISEL-LABEL: isnan_v8f32: @@ -1163,46 +1096,38 @@ ; GFX11SELDAG-LABEL: isnan_v16f32: ; GFX11SELDAG: ; %bb.0: ; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v4, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v5, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v6, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v7, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v8, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v8, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v9, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v10, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v11, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v12, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v13, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v14, 3 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0 ; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v15, 3 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v15, 0, 1, s0 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s1, v14, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s2, v13, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s3, v12, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s4, v11, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s5, v10, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s6, v9, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s7, v8, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s8, v7, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s9, v6, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s10, v5, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s11, v0, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s12, v1, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s13, v2, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s14, v3, 3 +; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s15, v4, 3 +; GFX11SELDAG-NEXT: v_cndmask_b16 v0.l, 0, 1, s11 +; GFX11SELDAG-NEXT: v_cndmask_b16 v1.l, 0, 1, s12 +; GFX11SELDAG-NEXT: v_cndmask_b16 v2.l, 0, 1, s13 +; GFX11SELDAG-NEXT: v_cndmask_b16 v3.l, 0, 1, s14 +; GFX11SELDAG-NEXT: v_cndmask_b16 v4.l, 0, 1, s15 +; GFX11SELDAG-NEXT: v_cndmask_b16 v5.l, 0, 1, s10 +; GFX11SELDAG-NEXT: v_cndmask_b16 v6.l, 0, 1, s9 +; GFX11SELDAG-NEXT: v_cndmask_b16 v7.l, 0, 1, s8 +; GFX11SELDAG-NEXT: v_cndmask_b16 v8.l, 0, 1, s7 +; GFX11SELDAG-NEXT: v_cndmask_b16 v9.l, 0, 1, s6 +; GFX11SELDAG-NEXT: v_cndmask_b16 v10.l, 0, 1, s5 +; GFX11SELDAG-NEXT: v_cndmask_b16 v11.l, 0, 1, s4 +; GFX11SELDAG-NEXT: v_cndmask_b16 v12.l, 0, 1, s3 +; GFX11SELDAG-NEXT: v_cndmask_b16 v13.l, 0, 1, s2 +; GFX11SELDAG-NEXT: v_cndmask_b16 v14.l, 0, 1, s1 +; GFX11SELDAG-NEXT: v_cndmask_b16 v15.l, 0, 1, s0 ; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31] ; ; GFX11GLISEL-LABEL: isnan_v16f32: @@ -1620,25 +1545,15 @@ ; GFX10CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s4 ; GFX10CHECK-NEXT: s_setpc_b64 s[30:31] ; -; GFX11SELDAG-LABEL: isnormal_v2f64: -; GFX11SELDAG: ; %bb.0: -; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11SELDAG-NEXT: v_cmp_class_f64_e64 s0, v[0:1], 0x108 -; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 -; GFX11SELDAG-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 0x108 -; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0 -; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31] -; -; GFX11GLISEL-LABEL: isnormal_v2f64: -; GFX11GLISEL: ; %bb.0: -; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11GLISEL-NEXT: v_cmp_class_f64_e64 s0, v[0:1], 0x108 -; GFX11GLISEL-NEXT: v_cmp_class_f64_e64 s1, v[2:3], 0x108 -; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0 -; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1 -; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31] +; GFX11CHECK-LABEL: isnormal_v2f64: +; GFX11CHECK: ; %bb.0: +; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11CHECK-NEXT: v_cmp_class_f64_e64 s0, v[0:1], 0x108 +; GFX11CHECK-NEXT: v_cmp_class_f64_e64 s1, v[2:3], 0x108 +; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11CHECK-NEXT: v_cndmask_b16 v0.l, 0, 1, s0 +; GFX11CHECK-NEXT: v_cndmask_b16 v1.l, 0, 1, s1 +; GFX11CHECK-NEXT: s_setpc_b64 s[30:31] %1 = call <2 x i1> @llvm.is.fpclass.v2f64(<2 x double> %x, i32 264) ; 0x108 = "normal" ret <2 x i1> %1 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll index fc4ac90..d9a085f 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
@@ -3821,20 +3821,35 @@ ; GFX900-GISEL-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31] ; -; GFX1100-SDAG-LABEL: v_log2_f32_from_fpext_bf16: -; GFX1100-SDAG: ; %bb.0: -; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX1100-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 -; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo -; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo -; GFX1100-SDAG-NEXT: v_ldexp_f32 v0, v0, v2 -; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1100-SDAG-NEXT: v_log_f32_e32 v0, v0 -; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0) -; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v0, v1 -; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31] +; GFX1100-SDAG-TRUE16-LABEL: v_log2_f32_from_fpext_bf16: +; GFX1100-SDAG-TRUE16: ; %bb.0: +; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1100-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 +; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo +; GFX1100-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo +; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1100-SDAG-TRUE16-NEXT: v_log_f32_e32 v0, v0 +; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0) +; GFX1100-SDAG-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX1100-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1100-SDAG-FAKE16-LABEL: v_log2_f32_from_fpext_bf16: +; GFX1100-SDAG-FAKE16: ; %bb.0: +; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1100-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1100-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 +; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo +; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo +; GFX1100-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v2 +; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1100-SDAG-FAKE16-NEXT: v_log_f32_e32 v0, v0 +; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0) +; GFX1100-SDAG-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX1100-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX1100-GISEL-LABEL: v_log2_f32_from_fpext_bf16: ; GFX1100-GISEL: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.round.ll b/llvm/test/CodeGen/AMDGPU/llvm.round.ll index 0fed9df..457016e 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.round.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.round.ll
@@ -761,6 +761,8 @@ ; R600-NEXT: ADD T0.X, T3.W, PV.W, ; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, ; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) +; R600-NEXT: ADD_INT * T2.X, PS, literal.x, +; R600-NEXT: 4(5.605194e-45), 0(0.000000e+00) %result = call <8 x float> @llvm.round.v8f32(<8 x float> %in) #1 store <8 x float> %result, ptr addrspace(1) %out ret void
diff --git a/llvm/test/CodeGen/AMDGPU/load-input-fold.ll b/llvm/test/CodeGen/AMDGPU/load-input-fold.ll index 7d34f6b..22742c1 100644 --- a/llvm/test/CodeGen/AMDGPU/load-input-fold.ll +++ b/llvm/test/CodeGen/AMDGPU/load-input-fold.ll
@@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 ;RUN: llc < %s -mtriple=r600 -mcpu=cayman define amdgpu_vs void @main(<4 x float> inreg %reg0, <4 x float> inreg %reg1, <4 x float> inreg %reg2, <4 x float> inreg %reg3) {
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll index 4ac8985..b582183 100644 --- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll
@@ -402,29 +402,17 @@ } define <3 x bfloat> @v_mad_mix_v3f32_clamp_precvt(<3 x bfloat> %src0, <3 x bfloat> %src1, <3 x bfloat> %src2) #0 { -; GFX1250-FAKE16-LABEL: v_mad_mix_v3f32_clamp_precvt: -; GFX1250-FAKE16: ; %bb.0: -; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v6, v0, v2, v4 op_sel_hi:[1,1,1] clamp -; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v0, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp -; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v1, v1, v3, v5 op_sel_hi:[1,1,1] clamp -; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0 -; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 -; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31] -; -; GFX1250-REAL16-LABEL: v_mad_mix_v3f32_clamp_precvt: -; GFX1250-REAL16: ; %bb.0: -; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0 -; GFX1250-REAL16-NEXT: v_fma_mix_f32_bf16 v1, v1, v3, v5 op_sel_hi:[1,1,1] clamp -; GFX1250-REAL16-NEXT: v_fma_mix_f32_bf16 v3, v0, v2, v4 op_sel_hi:[1,1,1] clamp -; GFX1250-REAL16-NEXT: v_fma_mix_f32_bf16 v0, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp -; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250-REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 -; GFX1250-REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v3, v0 -; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31] +; GFX1250-LABEL: v_mad_mix_v3f32_clamp_precvt: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mix_f32_bf16 v6, v0, v2, v4 op_sel_hi:[1,1,1] clamp +; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp +; GFX1250-NEXT: v_fma_mix_f32_bf16 v1, v1, v3, v5 op_sel_hi:[1,1,1] clamp +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0 +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %src0.ext = fpext <3 x bfloat> %src0 to <3 x float> %src1.ext = fpext <3 x bfloat> %src1 to <3 x float> %src2.ext = fpext <3 x bfloat> %src2 to <3 x float>
diff --git a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll index c90d990..b04cf6b 100644 --- a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
@@ -107,11 +107,8 @@ ; GFX11-TRUE16-LABEL: v_maximumnum_bf16: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 @@ -162,11 +159,8 @@ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 @@ -293,19 +287,16 @@ ; GFX11-TRUE16-LABEL: v_maximumnum_bf16_nnan: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2 ; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2 -; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 +; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -332,20 +323,17 @@ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2 -; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 +; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) ; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -12542,11 +12530,8 @@ ; GFX11-TRUE16-LABEL: v_maximumnum_bf16_no_ieee: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 @@ -12597,11 +12582,8 @@ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
diff --git a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll index 7615f66..6289e43 100644 --- a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
@@ -109,11 +109,8 @@ ; GFX11-TRUE16-LABEL: v_minimumnum_bf16: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 @@ -164,11 +161,8 @@ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 @@ -298,19 +292,16 @@ ; GFX11-TRUE16-LABEL: v_minimumnum_bf16_nnan: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2 ; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2 -; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 +; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -337,20 +328,17 @@ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2 -; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 +; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0) ; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -12577,11 +12565,8 @@ ; GFX11-TRUE16-LABEL: v_minimumnum_bf16_no_ieee: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 @@ -12632,11 +12617,8 @@ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
diff --git a/llvm/test/CodeGen/AMDGPU/misaligned-vgpr-regsequence.mir b/llvm/test/CodeGen/AMDGPU/misaligned-vgpr-regsequence.mir index fe2b51b..9f471e7 100644 --- a/llvm/test/CodeGen/AMDGPU/misaligned-vgpr-regsequence.mir +++ b/llvm/test/CodeGen/AMDGPU/misaligned-vgpr-regsequence.mir
@@ -1,3 +1,4 @@ +# NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 # RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-after=si-load-store-opt %s -o - | FileCheck %s # CHECK: misaligned_regsequence: @@ -30,3 +31,5 @@ FLAT_STORE_DWORDX3 %3, killed %5, 0, 0, implicit $exec, implicit $flat_scr :: (store (s96), align 4) S_ENDPGM 0 ... +## NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +# CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/pv.ll b/llvm/test/CodeGen/AMDGPU/pv.ll index 96db6ec..aabace6 100644 --- a/llvm/test/CodeGen/AMDGPU/pv.ll +++ b/llvm/test/CodeGen/AMDGPU/pv.ll
@@ -1,8 +1,82 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 ; RUN: llc -mtriple=r600 < %s | FileCheck %s ; CHECK: DOT4 * T{{[0-9]\.W}} (MASKED) ; CHECK: MAX T{{[0-9].[XYZW]}}, 0.0, PV.X define amdgpu_vs void @main(<4 x float> inreg %reg0, <4 x float> inreg %reg1, <4 x float> inreg %reg2, <4 x float> inreg %reg3, <4 x float> inreg %reg4, <4 x float> inreg %reg5, <4 x float> inreg %reg6, <4 x float> inreg %reg7) { +; CHECK-LABEL: main: +; CHECK: ; %bb.0: ; %main_body +; CHECK-NEXT: CALL_FS +; CHECK-NEXT: ALU 63, @6, KC0[CB0:0-32], KC1[] +; CHECK-NEXT: EXPORT T3.XYZW +; CHECK-NEXT: EXPORT T4.XYZW +; CHECK-NEXT: CF_END +; CHECK-NEXT: PAD +; CHECK-NEXT: ALU clause starting at 6: +; CHECK-NEXT: DOT4 T0.X, T2.X, T2.X, +; CHECK-NEXT: DOT4 T0.Y (MASKED), T2.Y, T2.Y, +; CHECK-NEXT: DOT4 T0.Z (MASKED), T2.Z, T2.Z, +; CHECK-NEXT: DOT4 * T0.W (MASKED), 0.0, 0.0, +; CHECK-NEXT: RECIPSQRT_CLAMPED * T0.X, |PV.X|, +; CHECK-NEXT: MUL_IEEE T2.X, T2.X, PS, +; CHECK-NEXT: MUL_IEEE T0.Y, T2.Y, PS, +; CHECK-NEXT: MUL_IEEE * T0.Z, T2.Z, PS, +; CHECK-NEXT: DOT4 T0.X, T2.X, KC0[7].X, +; CHECK-NEXT: DOT4 T0.Y (MASKED), T0.Y, KC0[7].Y, +; CHECK-NEXT: DOT4 T0.Z (MASKED), T0.Z, KC0[7].Z, +; CHECK-NEXT: DOT4 * T0.W (MASKED), 0.0, 0.0, +; CHECK-NEXT: MAX T0.W, 0.0, PV.X, +; CHECK-NEXT: MUL_IEEE * T2.W, T1.X, KC0[0].Z, +; CHECK-NEXT: MULADD_IEEE T2.Y, T1.Y, KC0[1].Z, PS, +; CHECK-NEXT: MULADD_IEEE T2.Z, KC0[4].Y, T3.Y, T6.Y, +; CHECK-NEXT: MULADD_IEEE T2.W, KC0[4].X, T3.X, T6.X, +; CHECK-NEXT: LOG_IEEE * T0.X, PV.W, +; CHECK-NEXT: MUL_IEEE T6.X, T1.X, KC0[0].W, +; CHECK-NEXT: MULADD_IEEE T6.Y, KC0[8].X, T3.X, PV.W, +; CHECK-NEXT: MULADD_IEEE T2.Z, KC0[8].Y, T3.Y, PV.Z, +; CHECK-NEXT: MUL NON-IEEE * T0.W, T7.X, PS, BS:VEC_201 +; CHECK-NEXT: MUL_IEEE * T2.W, T1.X, KC0[0].X, +; CHECK-NEXT: MULADD_IEEE * T2.W, T1.Y, KC0[1].X, PV.W, +; CHECK-NEXT: DOT4 T0.X, T2.X, KC0[5].X, +; CHECK-NEXT: DOT4 T0.Y (MASKED), T0.Y, KC0[5].Y, +; CHECK-NEXT: DOT4 T0.Z (MASKED), T0.Z, KC0[5].Z, +; CHECK-NEXT: DOT4 * T0.W (MASKED), 0.0, 0.0, +; CHECK-NEXT: MUL_IEEE T2.X, KC0[9].X, T4.X, +; CHECK-NEXT: MAX T0.Y, 0.0, PV.X, +; CHECK-NEXT: MUL_IEEE T0.Z, KC0[9].Y, T4.Y, +; CHECK-NEXT: MULADD_IEEE T2.W, T1.Z, KC0[2].X, T2.W, BS:VEC_021/SCL_122 +; CHECK-NEXT: EXP_IEEE * T0.W, T0.W, +; CHECK-NEXT: MULADD_IEEE T3.X, T1.W, KC0[3].X, PV.W, +; CHECK-NEXT: MUL_IEEE T3.Y, T1.X, KC0[0].Y, +; CHECK-NEXT: CNDGE T7.Z, T0.X, PS, 0.0, BS:VEC_120/SCL_212 +; CHECK-NEXT: MULADD_IEEE * T0.W, PV.Y, PV.Z, T2.Z, +; CHECK-NEXT: MUL_IEEE * T2.W, KC0[10].Y, T5.Y, +; CHECK-NEXT: MULADD_IEEE T0.X, T7.Z, PV.W, T0.W, +; CHECK-NEXT: MULADD_IEEE T4.Y, KC0[4].Z, T3.Z, T6.Z, +; CHECK-NEXT: MULADD_IEEE * T0.Z, T1.Y, KC0[1].Y, T3.Y, +; CHECK-NEXT: MULADD_IEEE T0.W, T0.Y, T2.X, T6.Y, BS:VEC_021/SCL_122 +; CHECK-NEXT: MUL_IEEE * T2.W, KC0[10].X, T5.X, +; CHECK-NEXT: MAX_DX10 T1.X, T4.W, 0.0, +; CHECK-NEXT: MULADD_IEEE T5.Y, T7.Z, PS, PV.W, +; CHECK-NEXT: MULADD_IEEE * T0.Z, T1.Z, KC0[2].Y, T0.Z, BS:VEC_102/SCL_221 +; CHECK-NEXT: MULADD_IEEE T0.W, KC0[8].Z, T3.Z, T4.Y, BS:VEC_021/SCL_122 +; CHECK-NEXT: MUL_IEEE * T2.W, KC0[9].Z, T4.Z, +; CHECK-NEXT: MULADD_IEEE T2.X, T0.Y, PS, PV.W, +; CHECK-NEXT: MULADD_IEEE T3.Y, T1.W, KC0[3].Y, T0.Z, +; CHECK-NEXT: MUL_IEEE T0.Z, KC0[10].Z, T5.Z, +; CHECK-NEXT: MAX_DX10 T0.W, T5.Y, 0.0, BS:VEC_120/SCL_212 +; CHECK-NEXT: MIN_DX10 * T4.W, T1.X, 1.0, +; CHECK-NEXT: MIN_DX10 T4.X, PV.W, 1.0, +; CHECK-NEXT: MULADD_IEEE T0.Y, T7.Z, PV.Z, PV.X, +; CHECK-NEXT: MULADD_IEEE T0.Z, T1.Y, KC0[1].W, T6.X, BS:VEC_021/SCL_122 +; CHECK-NEXT: MULADD_IEEE T0.W, T1.Z, KC0[2].Z, T2.Y, BS:VEC_102/SCL_221 +; CHECK-NEXT: MAX_DX10 * T2.W, T0.X, 0.0, +; CHECK-NEXT: MIN_DX10 T4.Y, PS, 1.0, +; CHECK-NEXT: MULADD_IEEE T3.Z, T1.W, KC0[3].Z, PV.W, +; CHECK-NEXT: MULADD_IEEE T0.W, T1.Z, KC0[2].W, PV.Z, +; CHECK-NEXT: MAX_DX10 * T2.W, PV.Y, 0.0, +; CHECK-NEXT: MIN_DX10 T4.Z, PS, 1.0, +; CHECK-NEXT: MULADD_IEEE * T3.W, T1.W, KC0[3].W, PV.W, main_body: %tmp = extractelement <4 x float> %reg1, i32 0 %tmp13 = extractelement <4 x float> %reg1, i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/r600.private-memory.ll b/llvm/test/CodeGen/AMDGPU/r600.private-memory.ll index 03e3712..11fcb05 100644 --- a/llvm/test/CodeGen/AMDGPU/r600.private-memory.ll +++ b/llvm/test/CodeGen/AMDGPU/r600.private-memory.ll
@@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 ; RUN: llc -mtriple=r600 -mcpu=cypress < %s | FileCheck %s -check-prefix=R600 -check-prefix=FUNC declare i32 @llvm.r600.read.tidig.x() nounwind readnone @@ -11,6 +12,23 @@ ; R600-NOT: MOV * TO.X define amdgpu_kernel void @work_item_info(ptr addrspace(1) %out, i32 %in) { +; R600-LABEL: work_item_info: +; R600: ; %bb.0: ; %entry +; R600-NEXT: ALU 9, @4, KC0[CB0:0-32], KC1[] +; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1 +; R600-NEXT: CF_END +; R600-NEXT: PAD +; R600-NEXT: ALU clause starting at 4: +; R600-NEXT: MOV T1.X, 0.0, +; R600-NEXT: MOV * T1.Y, 1, +; R600-NEXT: MOV * T0.Y, PV.X, +; R600-NEXT: MOV T1.Y, T1.Y, +; R600-NEXT: MOV * T0.W, KC0[2].Z, +; R600-NEXT: MOVA_INT * AR.x (MASKED), PV.W, +; R600-NEXT: MOV * T0.Y, T(0 + AR.x).Y+, +; R600-NEXT: ADD_INT T0.X, PV.Y, T0.X, +; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, +; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00) entry: %0 = alloca [2 x i32], addrspace(5) %1 = getelementptr [2 x i32], ptr addrspace(5) %0, i32 0, i32 1 @@ -23,3 +41,5 @@ store i32 %5, ptr addrspace(1) %out ret void } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; FUNC: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-trackers.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-trackers.ll index 7044517..4950de4 100644 --- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-trackers.ll +++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-trackers.ll
@@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 ; RUN: llc -mtriple=amdgpu11.00--amdpal -verify-misched < %s | FileCheck --check-prefixes=GFX11-PAL %s ; RUN: llc -mtriple=amdgpu11.00--amdpal -amdgpu-use-amdgpu-trackers=1 -verify-misched < %s | FileCheck --check-prefixes=GFX11-PAL-GCNTRACKERS %s ; RUN: llc -mtriple=amdgpu8.02 -amdgpu-scalarize-global-loads=false -verify-misched < %s | FileCheck --check-prefixes=TONGA %s @@ -29,10 +30,3548 @@ define amdgpu_gfx <72 x i32> @return_72xi32(<72 x i32> %val) #1 { +; GFX11-PAL-LABEL: return_72xi32: +; GFX11-PAL: ; %bb.0: +; GFX11-PAL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-PAL-NEXT: s_clause 0xc ; 52-byte Folded Spill +; GFX11-PAL-NEXT: scratch_store_b32 off, v40, s32 offset:212 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v41, s32 offset:208 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v42, s32 offset:204 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v43, s32 offset:200 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v44, s32 offset:196 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v56, s32 offset:192 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v57, s32 offset:188 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v58, s32 offset:184 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v59, s32 offset:180 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v60, s32 offset:176 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v61, s32 offset:172 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v62, s32 offset:168 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v63, s32 offset:164 +; GFX11-PAL-NEXT: s_clause 0x11 +; GFX11-PAL-NEXT: scratch_load_b32 v36, off, s32 offset:16 +; GFX11-PAL-NEXT: scratch_load_b32 v35, off, s32 offset:12 +; GFX11-PAL-NEXT: scratch_load_b32 v34, off, s32 offset:8 +; GFX11-PAL-NEXT: scratch_load_b32 v51, off, s32 offset:32 +; GFX11-PAL-NEXT: scratch_load_b32 v50, off, s32 offset:28 +; GFX11-PAL-NEXT: scratch_load_b32 v49, off, s32 offset:24 +; GFX11-PAL-NEXT: scratch_load_b32 v55, off, s32 offset:48 +; GFX11-PAL-NEXT: scratch_load_b32 v54, off, s32 offset:44 +; GFX11-PAL-NEXT: scratch_load_b32 v53, off, s32 offset:40 +; GFX11-PAL-NEXT: scratch_load_b32 v40, off, s32 offset:64 +; GFX11-PAL-NEXT: scratch_load_b32 v39, off, s32 offset:60 +; GFX11-PAL-NEXT: scratch_load_b32 v38, off, s32 offset:56 +; GFX11-PAL-NEXT: scratch_load_b32 v44, off, s32 offset:80 +; GFX11-PAL-NEXT: scratch_load_b32 v43, off, s32 offset:76 +; GFX11-PAL-NEXT: scratch_load_b32 v42, off, s32 offset:72 +; GFX11-PAL-NEXT: scratch_load_b32 v59, off, s32 offset:96 +; GFX11-PAL-NEXT: scratch_load_b32 v58, off, s32 offset:92 +; GFX11-PAL-NEXT: scratch_load_b32 v57, off, s32 offset:88 +; GFX11-PAL-NEXT: s_clause 0x1 +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[21:24], off offset:80 +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[17:20], off offset:64 +; GFX11-PAL-NEXT: s_clause 0x5 +; GFX11-PAL-NEXT: scratch_load_b32 v23, off, s32 offset:112 +; GFX11-PAL-NEXT: scratch_load_b32 v22, off, s32 offset:108 +; GFX11-PAL-NEXT: scratch_load_b32 v21, off, s32 offset:104 +; GFX11-PAL-NEXT: scratch_load_b32 v19, off, s32 offset:128 +; GFX11-PAL-NEXT: scratch_load_b32 v18, off, s32 offset:124 +; GFX11-PAL-NEXT: scratch_load_b32 v17, off, s32 offset:120 +; GFX11-PAL-NEXT: s_clause 0x1 +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[13:16], off offset:48 +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[9:12], off offset:32 +; GFX11-PAL-NEXT: s_clause 0x10 +; GFX11-PAL-NEXT: scratch_load_b32 v15, off, s32 offset:144 +; GFX11-PAL-NEXT: scratch_load_b32 v14, off, s32 offset:140 +; GFX11-PAL-NEXT: scratch_load_b32 v13, off, s32 offset:136 +; GFX11-PAL-NEXT: scratch_load_b32 v63, off, s32 offset:160 +; GFX11-PAL-NEXT: scratch_load_b32 v62, off, s32 offset:156 +; GFX11-PAL-NEXT: scratch_load_b32 v61, off, s32 offset:152 +; GFX11-PAL-NEXT: scratch_load_b32 v60, off, s32 offset:148 +; GFX11-PAL-NEXT: scratch_load_b32 v12, off, s32 offset:132 +; GFX11-PAL-NEXT: scratch_load_b32 v16, off, s32 offset:116 +; GFX11-PAL-NEXT: scratch_load_b32 v20, off, s32 offset:100 +; GFX11-PAL-NEXT: scratch_load_b32 v56, off, s32 offset:84 +; GFX11-PAL-NEXT: scratch_load_b32 v41, off, s32 offset:68 +; GFX11-PAL-NEXT: scratch_load_b32 v37, off, s32 offset:52 +; GFX11-PAL-NEXT: scratch_load_b32 v52, off, s32 offset:36 +; GFX11-PAL-NEXT: scratch_load_b32 v48, off, s32 offset:20 +; GFX11-PAL-NEXT: scratch_load_b32 v33, off, s32 offset:4 +; GFX11-PAL-NEXT: scratch_load_b32 v32, off, s32 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(10) +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[60:63], off offset:272 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(9) +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[12:15], off offset:256 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(8) +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[16:19], off offset:240 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(7) +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[20:23], off offset:224 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(6) +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[56:59], off offset:208 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(5) +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[41:44], off offset:192 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(4) +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[37:40], off offset:176 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(3) +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[52:55], off offset:160 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(2) +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[48:51], off offset:144 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(1) +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[33:36], off offset:128 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-NEXT: s_clause 0x3 +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[29:32], off offset:112 +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[25:28], off offset:96 +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 +; GFX11-PAL-NEXT: scratch_store_b128 v0, v[1:4], off +; GFX11-PAL-NEXT: s_clause 0xc ; 52-byte Folded Reload +; GFX11-PAL-NEXT: scratch_load_b32 v63, off, s32 offset:164 +; GFX11-PAL-NEXT: scratch_load_b32 v62, off, s32 offset:168 +; GFX11-PAL-NEXT: scratch_load_b32 v61, off, s32 offset:172 +; GFX11-PAL-NEXT: scratch_load_b32 v60, off, s32 offset:176 +; GFX11-PAL-NEXT: scratch_load_b32 v59, off, s32 offset:180 +; GFX11-PAL-NEXT: scratch_load_b32 v58, off, s32 offset:184 +; GFX11-PAL-NEXT: scratch_load_b32 v57, off, s32 offset:188 +; GFX11-PAL-NEXT: scratch_load_b32 v56, off, s32 offset:192 +; GFX11-PAL-NEXT: scratch_load_b32 v44, off, s32 offset:196 +; GFX11-PAL-NEXT: scratch_load_b32 v43, off, s32 offset:200 +; GFX11-PAL-NEXT: scratch_load_b32 v42, off, s32 offset:204 +; GFX11-PAL-NEXT: scratch_load_b32 v41, off, s32 offset:208 +; GFX11-PAL-NEXT: scratch_load_b32 v40, off, s32 offset:212 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-PAL-GCNTRACKERS-LABEL: return_72xi32: +; GFX11-PAL-GCNTRACKERS: ; %bb.0: +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0xf ; 64-byte Folded Spill +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v40, s32 offset:224 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v41, s32 offset:220 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v42, s32 offset:216 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v43, s32 offset:212 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v44, s32 offset:208 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v45, s32 offset:204 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v46, s32 offset:200 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v47, s32 offset:196 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v56, s32 offset:192 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v57, s32 offset:188 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v58, s32 offset:184 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v59, s32 offset:180 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v60, s32 offset:176 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v61, s32 offset:172 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v62, s32 offset:168 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v63, s32 offset:164 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v47, v31 :: v_dual_mov_b32 v46, v30 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v45, v29 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[25:28], s32 offset:228 ; 16-byte Folded Spill +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v28, v8 :: v_dual_mov_b32 v27, v7 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v26, v6 :: v_dual_mov_b32 v25, v5 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v32, v4 :: v_dual_mov_b32 v31, v3 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v30, v2 :: v_dual_mov_b32 v29, v1 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x1b +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v36, off, s32 offset:16 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v35, off, s32 offset:12 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v34, off, s32 offset:8 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v33, off, s32 offset:4 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v51, off, s32 offset:32 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v50, off, s32 offset:28 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v49, off, s32 offset:24 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v55, off, s32 offset:48 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v54, off, s32 offset:44 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v53, off, s32 offset:40 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v40, off, s32 offset:64 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v39, off, s32 offset:60 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v38, off, s32 offset:56 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v44, off, s32 offset:80 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v43, off, s32 offset:76 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v42, off, s32 offset:72 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v59, off, s32 offset:96 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v58, off, s32 offset:92 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v57, off, s32 offset:88 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v56, off, s32 offset:84 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v63, off, s32 offset:112 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v62, off, s32 offset:108 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v61, off, s32 offset:104 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v8, off, s32 offset:128 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v7, off, s32 offset:124 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v6, off, s32 offset:120 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v4, off, s32 offset:144 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v3, off, s32 offset:140 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x1 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[21:24], off offset:80 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[17:20], off offset:64 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0xb +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v2, off, s32 offset:136 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v23, off, s32 offset:160 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v22, off, s32 offset:156 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v21, off, s32 offset:152 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v20, off, s32 offset:148 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v1, off, s32 offset:132 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v5, off, s32 offset:116 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v60, off, s32 offset:100 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v41, off, s32 offset:68 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v37, off, s32 offset:52 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v52, off, s32 offset:36 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v48, off, s32 offset:20 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x1 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[13:16], off offset:48 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[9:12], off offset:32 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v12, off, s32 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v9, v45 :: v_dual_mov_b32 v10, v46 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v11, v47 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(8) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[20:23], off offset:272 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(7) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[1:4], off offset:256 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(6) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[5:8], off offset:240 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(5) +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x1 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[60:63], off offset:224 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[56:59], off offset:208 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(4) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[41:44], off offset:192 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(3) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[37:40], off offset:176 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(2) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[52:55], off offset:160 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(1) +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x1 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[48:51], off offset:144 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[33:36], off offset:128 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[9:12], off offset:112 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[1:4], off, s32 offset:228 ; 16-byte Folded Reload +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x2 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[1:4], off offset:96 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[25:28], off offset:16 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 v0, v[29:32], off +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0xf ; 64-byte Folded Reload +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v63, off, s32 offset:164 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v62, off, s32 offset:168 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v61, off, s32 offset:172 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v60, off, s32 offset:176 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v59, off, s32 offset:180 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v58, off, s32 offset:184 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v57, off, s32 offset:188 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v56, off, s32 offset:192 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v47, off, s32 offset:196 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v46, off, s32 offset:200 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v45, off, s32 offset:204 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v44, off, s32 offset:208 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v43, off, s32 offset:212 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v42, off, s32 offset:216 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v41, off, s32 offset:220 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v40, off, s32 offset:224 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31] +; +; TONGA-LABEL: return_72xi32: +; TONGA: ; %bb.0: +; TONGA-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:160 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:284 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:156 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:280 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:152 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:276 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:148 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:272 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:268 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:264 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:136 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:260 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:132 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:256 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:252 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:248 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:244 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:240 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:236 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:232 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:228 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:224 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:220 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:216 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:212 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:208 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:204 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:200 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:196 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:192 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:188 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:184 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:180 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:176 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:172 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:168 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:164 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:160 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:156 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:152 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:148 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:144 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:140 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:136 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:132 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:128 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s32 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:124 +; TONGA-NEXT: buffer_store_dword v31, v0, s[0:3], 0 offen offset:120 +; TONGA-NEXT: buffer_store_dword v30, v0, s[0:3], 0 offen offset:116 +; TONGA-NEXT: buffer_store_dword v29, v0, s[0:3], 0 offen offset:112 +; TONGA-NEXT: buffer_store_dword v28, v0, s[0:3], 0 offen offset:108 +; TONGA-NEXT: buffer_store_dword v27, v0, s[0:3], 0 offen offset:104 +; TONGA-NEXT: buffer_store_dword v26, v0, s[0:3], 0 offen offset:100 +; TONGA-NEXT: buffer_store_dword v25, v0, s[0:3], 0 offen offset:96 +; TONGA-NEXT: buffer_store_dword v24, v0, s[0:3], 0 offen offset:92 +; TONGA-NEXT: buffer_store_dword v23, v0, s[0:3], 0 offen offset:88 +; TONGA-NEXT: buffer_store_dword v22, v0, s[0:3], 0 offen offset:84 +; TONGA-NEXT: buffer_store_dword v21, v0, s[0:3], 0 offen offset:80 +; TONGA-NEXT: buffer_store_dword v20, v0, s[0:3], 0 offen offset:76 +; TONGA-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen offset:72 +; TONGA-NEXT: buffer_store_dword v18, v0, s[0:3], 0 offen offset:68 +; TONGA-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:64 +; TONGA-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:60 +; TONGA-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:56 +; TONGA-NEXT: buffer_store_dword v14, v0, s[0:3], 0 offen offset:52 +; TONGA-NEXT: buffer_store_dword v13, v0, s[0:3], 0 offen offset:48 +; TONGA-NEXT: buffer_store_dword v12, v0, s[0:3], 0 offen offset:44 +; TONGA-NEXT: buffer_store_dword v11, v0, s[0:3], 0 offen offset:40 +; TONGA-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:36 +; TONGA-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:32 +; TONGA-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:28 +; TONGA-NEXT: buffer_store_dword v7, v0, s[0:3], 0 offen offset:24 +; TONGA-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:20 +; TONGA-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:16 +; TONGA-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen offset:12 +; TONGA-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:8 +; TONGA-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 +; TONGA-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: s_setpc_b64 s[30:31] +; +; TONGA-GCNTRACKERS-LABEL: return_72xi32: +; TONGA-GCNTRACKERS: ; %bb.0: +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:160 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:284 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:156 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:280 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:152 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:276 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:148 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:272 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:268 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:264 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:136 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:260 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:132 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:256 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:252 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:248 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:244 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:240 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:236 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:232 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:228 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:224 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:220 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:216 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:212 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:208 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:204 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:200 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:196 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:192 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:188 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:184 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:180 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:176 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:172 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:168 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:164 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:160 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:156 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:152 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:148 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:144 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:140 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:136 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:132 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:128 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:124 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v31, v0, s[0:3], 0 offen offset:120 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v30, v0, s[0:3], 0 offen offset:116 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v29, v0, s[0:3], 0 offen offset:112 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v28, v0, s[0:3], 0 offen offset:108 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v27, v0, s[0:3], 0 offen offset:104 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v26, v0, s[0:3], 0 offen offset:100 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v25, v0, s[0:3], 0 offen offset:96 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v24, v0, s[0:3], 0 offen offset:92 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v23, v0, s[0:3], 0 offen offset:88 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v22, v0, s[0:3], 0 offen offset:84 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v21, v0, s[0:3], 0 offen offset:80 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v20, v0, s[0:3], 0 offen offset:76 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen offset:72 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v18, v0, s[0:3], 0 offen offset:68 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:64 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:60 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:56 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v14, v0, s[0:3], 0 offen offset:52 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v13, v0, s[0:3], 0 offen offset:48 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v12, v0, s[0:3], 0 offen offset:44 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v11, v0, s[0:3], 0 offen offset:40 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:36 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:32 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:28 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v7, v0, s[0:3], 0 offen offset:24 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:20 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:16 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen offset:12 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:8 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31] +; +; GFX908-LABEL: return_72xi32: +; GFX908: ; %bb.0: +; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:160 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:284 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:156 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:280 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:152 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:276 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:148 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:272 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:268 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:264 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:136 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:260 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:132 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:256 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:252 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:248 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:244 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:240 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:236 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:232 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:228 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:224 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:220 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:216 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:212 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:208 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:204 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:200 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:196 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:192 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:188 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:184 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:180 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:176 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:172 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:168 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:164 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:160 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:156 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:152 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:148 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:144 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:140 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:136 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:132 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:128 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s32 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:124 +; GFX908-NEXT: buffer_store_dword v31, v0, s[0:3], 0 offen offset:120 +; GFX908-NEXT: buffer_store_dword v30, v0, s[0:3], 0 offen offset:116 +; GFX908-NEXT: buffer_store_dword v29, v0, s[0:3], 0 offen offset:112 +; GFX908-NEXT: buffer_store_dword v28, v0, s[0:3], 0 offen offset:108 +; GFX908-NEXT: buffer_store_dword v27, v0, s[0:3], 0 offen offset:104 +; GFX908-NEXT: buffer_store_dword v26, v0, s[0:3], 0 offen offset:100 +; GFX908-NEXT: buffer_store_dword v25, v0, s[0:3], 0 offen offset:96 +; GFX908-NEXT: buffer_store_dword v24, v0, s[0:3], 0 offen offset:92 +; GFX908-NEXT: buffer_store_dword v23, v0, s[0:3], 0 offen offset:88 +; GFX908-NEXT: buffer_store_dword v22, v0, s[0:3], 0 offen offset:84 +; GFX908-NEXT: buffer_store_dword v21, v0, s[0:3], 0 offen offset:80 +; GFX908-NEXT: buffer_store_dword v20, v0, s[0:3], 0 offen offset:76 +; GFX908-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen offset:72 +; GFX908-NEXT: buffer_store_dword v18, v0, s[0:3], 0 offen offset:68 +; GFX908-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:64 +; GFX908-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:60 +; GFX908-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:56 +; GFX908-NEXT: buffer_store_dword v14, v0, s[0:3], 0 offen offset:52 +; GFX908-NEXT: buffer_store_dword v13, v0, s[0:3], 0 offen offset:48 +; GFX908-NEXT: buffer_store_dword v12, v0, s[0:3], 0 offen offset:44 +; GFX908-NEXT: buffer_store_dword v11, v0, s[0:3], 0 offen offset:40 +; GFX908-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:36 +; GFX908-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:32 +; GFX908-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:28 +; GFX908-NEXT: buffer_store_dword v7, v0, s[0:3], 0 offen offset:24 +; GFX908-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:20 +; GFX908-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:16 +; GFX908-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen offset:12 +; GFX908-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:8 +; GFX908-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 +; GFX908-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: s_setpc_b64 s[30:31] +; +; GFX908-GCNTRACKERS-LABEL: return_72xi32: +; GFX908-GCNTRACKERS: ; %bb.0: +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:160 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:284 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:156 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:280 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:152 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:276 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:148 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:272 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:268 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:264 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:136 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:260 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:132 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:256 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:252 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:248 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:244 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:240 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:236 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:232 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:228 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:224 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:220 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:216 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:212 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:208 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:204 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:200 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:196 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:192 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:188 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:184 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:180 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:176 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:172 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:168 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:164 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:160 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:156 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:152 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:148 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:144 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:140 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:136 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:132 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:128 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:124 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v31, v0, s[0:3], 0 offen offset:120 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v30, v0, s[0:3], 0 offen offset:116 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v29, v0, s[0:3], 0 offen offset:112 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v28, v0, s[0:3], 0 offen offset:108 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v27, v0, s[0:3], 0 offen offset:104 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v26, v0, s[0:3], 0 offen offset:100 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v25, v0, s[0:3], 0 offen offset:96 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v24, v0, s[0:3], 0 offen offset:92 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v23, v0, s[0:3], 0 offen offset:88 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v22, v0, s[0:3], 0 offen offset:84 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v21, v0, s[0:3], 0 offen offset:80 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v20, v0, s[0:3], 0 offen offset:76 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen offset:72 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v18, v0, s[0:3], 0 offen offset:68 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:64 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:60 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:56 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v14, v0, s[0:3], 0 offen offset:52 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v13, v0, s[0:3], 0 offen offset:48 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v12, v0, s[0:3], 0 offen offset:44 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v11, v0, s[0:3], 0 offen offset:40 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:36 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:32 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:28 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v7, v0, s[0:3], 0 offen offset:24 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:20 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:16 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen offset:12 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:8 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31] +; +; GENERIC-LABEL: return_72xi32: +; GENERIC: ; %bb.0: +; GENERIC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:160 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:284 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:156 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:280 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:152 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:276 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:148 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:272 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:268 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:264 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:136 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:260 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:132 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:256 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:252 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:248 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:244 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:240 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:236 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:232 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:228 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:224 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:220 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:216 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:212 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:208 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:204 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:200 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:196 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:192 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:188 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:184 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:180 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:176 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:172 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:168 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:164 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:160 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:156 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:152 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:148 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:144 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:140 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:136 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:132 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:128 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s32 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:124 +; GENERIC-NEXT: buffer_store_dword v31, v0, s[0:3], 0 offen offset:120 +; GENERIC-NEXT: buffer_store_dword v30, v0, s[0:3], 0 offen offset:116 +; GENERIC-NEXT: buffer_store_dword v29, v0, s[0:3], 0 offen offset:112 +; GENERIC-NEXT: buffer_store_dword v28, v0, s[0:3], 0 offen offset:108 +; GENERIC-NEXT: buffer_store_dword v27, v0, s[0:3], 0 offen offset:104 +; GENERIC-NEXT: buffer_store_dword v26, v0, s[0:3], 0 offen offset:100 +; GENERIC-NEXT: buffer_store_dword v25, v0, s[0:3], 0 offen offset:96 +; GENERIC-NEXT: buffer_store_dword v24, v0, s[0:3], 0 offen offset:92 +; GENERIC-NEXT: buffer_store_dword v23, v0, s[0:3], 0 offen offset:88 +; GENERIC-NEXT: buffer_store_dword v22, v0, s[0:3], 0 offen offset:84 +; GENERIC-NEXT: buffer_store_dword v21, v0, s[0:3], 0 offen offset:80 +; GENERIC-NEXT: buffer_store_dword v20, v0, s[0:3], 0 offen offset:76 +; GENERIC-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen offset:72 +; GENERIC-NEXT: buffer_store_dword v18, v0, s[0:3], 0 offen offset:68 +; GENERIC-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:64 +; GENERIC-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:60 +; GENERIC-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:56 +; GENERIC-NEXT: buffer_store_dword v14, v0, s[0:3], 0 offen offset:52 +; GENERIC-NEXT: buffer_store_dword v13, v0, s[0:3], 0 offen offset:48 +; GENERIC-NEXT: buffer_store_dword v12, v0, s[0:3], 0 offen offset:44 +; GENERIC-NEXT: buffer_store_dword v11, v0, s[0:3], 0 offen offset:40 +; GENERIC-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:36 +; GENERIC-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:32 +; GENERIC-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:28 +; GENERIC-NEXT: buffer_store_dword v7, v0, s[0:3], 0 offen offset:24 +; GENERIC-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:20 +; GENERIC-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:16 +; GENERIC-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen offset:12 +; GENERIC-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:8 +; GENERIC-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 +; GENERIC-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen +; GENERIC-NEXT: s_waitcnt vmcnt(0) expcnt(0) +; GENERIC-NEXT: s_setpc_b64 s[30:31] +; +; GENERIC-GCNTRACKERS-LABEL: return_72xi32: +; GENERIC-GCNTRACKERS: ; %bb.0: +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:160 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:284 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:156 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:280 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:152 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:276 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:148 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:272 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:268 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:264 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:136 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:260 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:132 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:256 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:252 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:248 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:244 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:240 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:236 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:232 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:228 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:224 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:220 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:216 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:212 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:208 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:204 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:200 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:196 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:192 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:188 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:184 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:180 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:176 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:172 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:168 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:164 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:160 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:156 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:152 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:148 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:144 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:140 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:136 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:132 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:128 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s32 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:124 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v31, v0, s[0:3], 0 offen offset:120 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v30, v0, s[0:3], 0 offen offset:116 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v29, v0, s[0:3], 0 offen offset:112 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v28, v0, s[0:3], 0 offen offset:108 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v27, v0, s[0:3], 0 offen offset:104 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v26, v0, s[0:3], 0 offen offset:100 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v25, v0, s[0:3], 0 offen offset:96 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v24, v0, s[0:3], 0 offen offset:92 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v23, v0, s[0:3], 0 offen offset:88 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v22, v0, s[0:3], 0 offen offset:84 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v21, v0, s[0:3], 0 offen offset:80 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v20, v0, s[0:3], 0 offen offset:76 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen offset:72 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v18, v0, s[0:3], 0 offen offset:68 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:64 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:60 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:56 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v14, v0, s[0:3], 0 offen offset:52 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v13, v0, s[0:3], 0 offen offset:48 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v12, v0, s[0:3], 0 offen offset:44 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v11, v0, s[0:3], 0 offen offset:40 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:36 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:32 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:28 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v7, v0, s[0:3], 0 offen offset:24 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:20 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:16 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen offset:12 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:8 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31] ret <72 x i32> %val } define amdgpu_gfx void @call_72xi32() #1 { +; GFX11-PAL-LABEL: call_72xi32: +; GFX11-PAL: ; %bb.0: ; %entry +; GFX11-PAL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-PAL-NEXT: s_mov_b32 s38, s33 +; GFX11-PAL-NEXT: s_add_i32 s33, s32, 0x1ff +; GFX11-PAL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-PAL-NEXT: s_and_b32 s33, s33, 0xfffffe00 +; GFX11-PAL-NEXT: s_or_saveexec_b32 s0, -1 +; GFX11-PAL-NEXT: scratch_store_b32 off, v62, s33 offset:1600 ; 4-byte Folded Spill +; GFX11-PAL-NEXT: s_mov_b32 exec_lo, s0 +; GFX11-PAL-NEXT: s_mov_b32 s39, s34 +; GFX11-PAL-NEXT: s_mov_b32 s34, s32 +; GFX11-PAL-NEXT: s_addk_i32 s32, 0xa00 +; GFX11-PAL-NEXT: s_clause 0xd ; 56-byte Folded Spill +; GFX11-PAL-NEXT: scratch_store_b32 off, v40, s33 offset:52 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v41, s33 offset:48 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v42, s33 offset:44 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v43, s33 offset:40 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v44, s33 offset:36 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v45, s33 offset:32 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v46, s33 offset:28 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v47, s33 offset:24 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v56, s33 offset:20 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v57, s33 offset:16 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v58, s33 offset:12 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v59, s33 offset:8 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v60, s33 offset:4 +; GFX11-PAL-NEXT: ; meta instruction +; GFX11-PAL-NEXT: scratch_store_b32 off, v61, s33 +; GFX11-PAL-NEXT: v_writelane_b32 v62, s30, 0 +; GFX11-PAL-NEXT: v_writelane_b32 v62, s31, 1 +; GFX11-PAL-NEXT: s_mov_b32 s0, 0 +; GFX11-PAL-NEXT: v_mov_b32_e32 v4, 0 +; GFX11-PAL-NEXT: s_mov_b32 s1, s0 +; GFX11-PAL-NEXT: s_mov_b32 s2, s0 +; GFX11-PAL-NEXT: s_mov_b32 s3, s0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 +; GFX11-PAL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 +; GFX11-PAL-NEXT: s_add_i32 s0, s32, 0xa0 +; GFX11-PAL-NEXT: s_add_i32 s1, s32, 0x90 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[0:3], s32 +; GFX11-PAL-NEXT: scratch_store_b32 off, v4, s0 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[0:3], s1 +; GFX11-PAL-NEXT: s_add_i32 s0, s32, 0x80 +; GFX11-PAL-NEXT: s_add_i32 s1, s32, 0x70 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[0:3], s0 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[0:3], s1 +; GFX11-PAL-NEXT: s_add_i32 s0, s32, 0x60 +; GFX11-PAL-NEXT: s_add_i32 s1, s32, 0x50 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[0:3], s0 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[0:3], s1 +; GFX11-PAL-NEXT: s_add_i32 s0, s32, 64 +; GFX11-PAL-NEXT: s_add_i32 s1, s32, 48 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[0:3], s0 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[0:3], s1 +; GFX11-PAL-NEXT: s_add_i32 s0, s32, 32 +; GFX11-PAL-NEXT: s_add_i32 s1, s32, 16 +; GFX11-PAL-NEXT: s_add_i32 s2, s33, 0x200 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[0:3], s0 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[0:3], s1 +; GFX11-PAL-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_mov_b32 v4, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v7, 0 :: v_dual_mov_b32 v6, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v8, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v11, 0 :: v_dual_mov_b32 v10, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v13, 0 :: v_dual_mov_b32 v12, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v15, 0 :: v_dual_mov_b32 v14, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v17, 0 :: v_dual_mov_b32 v16, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v19, 0 :: v_dual_mov_b32 v18, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v21, 0 :: v_dual_mov_b32 v20, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v23, 0 :: v_dual_mov_b32 v22, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v25, 0 :: v_dual_mov_b32 v24, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v27, 0 :: v_dual_mov_b32 v26, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v29, 0 :: v_dual_mov_b32 v28, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v31, 0 :: v_dual_mov_b32 v30, 0 +; GFX11-PAL-NEXT: s_mov_b32 s1, return_72xi32@abs32@hi +; GFX11-PAL-NEXT: s_mov_b32 s0, return_72xi32@abs32@lo +; GFX11-PAL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-PAL-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-PAL-NEXT: s_clause 0xb +; GFX11-PAL-NEXT: scratch_load_b128 v[43:46], off, s33 offset:624 +; GFX11-PAL-NEXT: scratch_load_b128 v[47:50], off, s33 offset:640 +; GFX11-PAL-NEXT: scratch_load_b128 v[16:19], off, s33 offset:784 +; GFX11-PAL-NEXT: scratch_load_b128 v[12:15], off, s33 offset:768 +; GFX11-PAL-NEXT: scratch_load_b128 v[8:11], off, s33 offset:752 +; GFX11-PAL-NEXT: scratch_load_b128 v[4:7], off, s33 offset:736 +; GFX11-PAL-NEXT: scratch_load_b128 v[0:3], off, s33 offset:720 +; GFX11-PAL-NEXT: scratch_load_b128 v[24:27], off, s33 offset:656 +; GFX11-PAL-NEXT: scratch_load_b128 v[36:39], off, s33 offset:704 +; GFX11-PAL-NEXT: scratch_load_b128 v[32:35], off, s33 offset:688 +; GFX11-PAL-NEXT: scratch_load_b128 v[28:31], off, s33 offset:672 +; GFX11-PAL-NEXT: scratch_load_b128 v[20:23], off, s33 offset:512 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(6) +; GFX11-PAL-NEXT: v_mov_b32_e32 v6, 24 +; GFX11-PAL-NEXT: s_add_i32 s2, s32, 0xa0 +; GFX11-PAL-NEXT: s_add_i32 s3, s32, 0x90 +; GFX11-PAL-NEXT: s_add_i32 s35, s32, 0x80 +; GFX11-PAL-NEXT: s_add_i32 s36, s32, 0x70 +; GFX11-PAL-NEXT: s_add_i32 s37, s32, 0x6c +; GFX11-PAL-NEXT: s_waitcnt vmcnt(4) +; GFX11-PAL-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v53, v26 +; GFX11-PAL-NEXT: v_dual_mov_b32 v41, v1 :: v_dual_mov_b32 v42, v2 +; GFX11-PAL-NEXT: v_mov_b32_e32 v51, v24 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1584 ; 16-byte Folded Spill +; GFX11-PAL-NEXT: scratch_load_b128 v[20:23], off, s33 offset:528 +; GFX11-PAL-NEXT: v_mov_b32_e32 v52, v25 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1536 ; 16-byte Folded Spill +; GFX11-PAL-NEXT: scratch_load_b128 v[20:23], off, s33 offset:544 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1568 ; 16-byte Folded Spill +; GFX11-PAL-NEXT: scratch_load_b128 v[20:23], off, s33 offset:560 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1552 ; 16-byte Folded Spill +; GFX11-PAL-NEXT: s_clause 0x2 +; GFX11-PAL-NEXT: scratch_load_b128 v[20:23], off, s33 offset:576 +; GFX11-PAL-NEXT: scratch_load_b128 v[58:61], off, s33 offset:592 +; GFX11-PAL-NEXT: scratch_load_b128 v[54:57], off, s33 offset:608 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[46:49], s32 +; GFX11-PAL-NEXT: scratch_store_b32 off, v19, s2 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[15:18], s3 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[11:14], s35 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[7:10], s36 +; GFX11-PAL-NEXT: s_add_i32 s2, s32, 0x60 +; GFX11-PAL-NEXT: scratch_store_b32 off, v6, s37 +; GFX11-PAL-NEXT: scratch_store_b96 off, v[3:5], s2 +; GFX11-PAL-NEXT: s_add_i32 s2, s32, 0x50 +; GFX11-PAL-NEXT: s_add_i32 s3, s32, 64 +; GFX11-PAL-NEXT: s_add_i32 s35, s32, 48 +; GFX11-PAL-NEXT: s_add_i32 s36, s32, 32 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[39:42], s2 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[35:38], s3 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[31:34], s35 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[27:30], s36 +; GFX11-PAL-NEXT: s_add_i32 s2, s32, 16 +; GFX11-PAL-NEXT: v_mov_b32_e32 v29, v43 +; GFX11-PAL-NEXT: scratch_store_b128 off, v[50:53], s2 +; GFX11-PAL-NEXT: s_clause 0x3 ; 64-byte Folded Reload +; GFX11-PAL-NEXT: scratch_load_b128 v[1:4], off, s33 offset:1584 +; GFX11-PAL-NEXT: scratch_load_b128 v[5:8], off, s33 offset:1536 +; GFX11-PAL-NEXT: scratch_load_b128 v[9:12], off, s33 offset:1568 +; GFX11-PAL-NEXT: scratch_load_b128 v[13:16], off, s33 offset:1552 +; GFX11-PAL-NEXT: s_add_i32 s2, s33, 0x400 +; GFX11-PAL-NEXT: v_dual_mov_b32 v30, v44 :: v_dual_mov_b32 v31, v45 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(3) +; GFX11-PAL-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 42 +; GFX11-PAL-NEXT: v_dual_mov_b32 v17, v20 :: v_dual_mov_b32 v18, v21 +; GFX11-PAL-NEXT: v_dual_mov_b32 v19, v22 :: v_dual_mov_b32 v20, v23 +; GFX11-PAL-NEXT: v_dual_mov_b32 v21, v58 :: v_dual_mov_b32 v22, v59 +; GFX11-PAL-NEXT: v_dual_mov_b32 v23, v60 :: v_dual_mov_b32 v24, v61 +; GFX11-PAL-NEXT: v_dual_mov_b32 v25, v54 :: v_dual_mov_b32 v26, v55 +; GFX11-PAL-NEXT: v_dual_mov_b32 v27, v56 :: v_dual_mov_b32 v28, v57 +; GFX11-PAL-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-PAL-NEXT: s_clause 0xd ; 56-byte Folded Reload +; GFX11-PAL-NEXT: scratch_load_b32 v61, off, s33 +; GFX11-PAL-NEXT: scratch_load_b32 v60, off, s33 offset:4 +; GFX11-PAL-NEXT: scratch_load_b32 v59, off, s33 offset:8 +; GFX11-PAL-NEXT: scratch_load_b32 v58, off, s33 offset:12 +; GFX11-PAL-NEXT: scratch_load_b32 v57, off, s33 offset:16 +; GFX11-PAL-NEXT: scratch_load_b32 v56, off, s33 offset:20 +; GFX11-PAL-NEXT: scratch_load_b32 v47, off, s33 offset:24 +; GFX11-PAL-NEXT: scratch_load_b32 v46, off, s33 offset:28 +; GFX11-PAL-NEXT: scratch_load_b32 v45, off, s33 offset:32 +; GFX11-PAL-NEXT: scratch_load_b32 v44, off, s33 offset:36 +; GFX11-PAL-NEXT: scratch_load_b32 v43, off, s33 offset:40 +; GFX11-PAL-NEXT: scratch_load_b32 v42, off, s33 offset:44 +; GFX11-PAL-NEXT: scratch_load_b32 v41, off, s33 offset:48 +; GFX11-PAL-NEXT: scratch_load_b32 v40, off, s33 offset:52 +; GFX11-PAL-NEXT: v_readlane_b32 s30, v62, 0 +; GFX11-PAL-NEXT: v_readlane_b32 s31, v62, 1 +; GFX11-PAL-NEXT: s_mov_b32 s32, s34 +; GFX11-PAL-NEXT: s_mov_b32 s34, s39 +; GFX11-PAL-NEXT: s_or_saveexec_b32 s0, -1 +; GFX11-PAL-NEXT: scratch_load_b32 v62, off, s33 offset:1600 ; 4-byte Folded Reload +; GFX11-PAL-NEXT: s_mov_b32 exec_lo, s0 +; GFX11-PAL-NEXT: s_mov_b32 s33, s38 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-PAL-GCNTRACKERS-LABEL: call_72xi32: +; GFX11-PAL-GCNTRACKERS: ; %bb.0: ; %entry +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 s35, s33 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s33, s32, 0x1ff +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s33, s33, 0xfffffe00 +; GFX11-PAL-GCNTRACKERS-NEXT: s_or_saveexec_b32 s0, -1 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v61, s33 offset:1916 ; 4-byte Folded Spill +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 s36, s34 +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 s34, s32 +; GFX11-PAL-GCNTRACKERS-NEXT: s_addk_i32 s32, 0xa00 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0xc ; 52-byte Folded Spill +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v40, s33 offset:48 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v41, s33 offset:44 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v42, s33 offset:40 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v43, s33 offset:36 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v44, s33 offset:32 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v45, s33 offset:28 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v46, s33 offset:24 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v47, s33 offset:20 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v56, s33 offset:16 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v57, s33 offset:12 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v58, s33 offset:8 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v59, s33 offset:4 +; GFX11-PAL-GCNTRACKERS-NEXT: ; meta instruction +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v60, s33 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v61, s30, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v61, s31, 1 +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 s0, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 s1, s0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 s2, s0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 s3, s0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s0, s32, 0xa0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 0x90 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[0:3], s32 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v4, s0 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[0:3], s1 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s0, s32, 0x80 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 0x70 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[0:3], s0 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[0:3], s1 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s0, s32, 0x60 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 0x50 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[0:3], s0 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[0:3], s1 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s0, s32, 64 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 48 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[0:3], s0 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[0:3], s1 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s0, s32, 32 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s1, s32, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s2, s33, 0x200 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[0:3], s0 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[0:3], s1 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_mov_b32 v4, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v7, 0 :: v_dual_mov_b32 v6, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v8, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v11, 0 :: v_dual_mov_b32 v10, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v13, 0 :: v_dual_mov_b32 v12, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v15, 0 :: v_dual_mov_b32 v14, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v17, 0 :: v_dual_mov_b32 v16, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v19, 0 :: v_dual_mov_b32 v18, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v21, 0 :: v_dual_mov_b32 v20, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v23, 0 :: v_dual_mov_b32 v22, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v25, 0 :: v_dual_mov_b32 v24, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v27, 0 :: v_dual_mov_b32 v26, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v29, 0 :: v_dual_mov_b32 v28, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v31, 0 :: v_dual_mov_b32 v30, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 s1, return_72xi32@abs32@hi +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 s0, return_72xi32@abs32@lo +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-PAL-GCNTRACKERS-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x1 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[0:3], off, s33 offset:624 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[4:7], off, s33 offset:640 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0xa0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v32, v7 :: v_dual_mov_b32 v31, v6 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v30, v5 :: v_dual_mov_b32 v29, v4 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v28, v3 :: v_dual_mov_b32 v27, v2 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v26, v1 :: v_dual_mov_b32 v25, v0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x5 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[9:12], off, s33 offset:736 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[1:4], off, s33 offset:512 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[21:24], off, s33 offset:784 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[17:20], off, s33 offset:768 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[13:16], off, s33 offset:752 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[5:8], off, s33 offset:720 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x7 ; 128-byte Folded Spill +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[5:8], s33 offset:1536 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[9:12], s33 offset:1552 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[13:16], s33 offset:1568 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[17:20], s33 offset:1584 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[21:24], s33 offset:1600 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[25:28], s33 offset:1616 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[29:32], s33 offset:1632 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[33:36], s33 offset:1648 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x1 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[12:15], off, s33 offset:704 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[8:11], off, s33 offset:688 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v0, 24 :: v_dual_mov_b32 v1, 42 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v16, v15 :: v_dual_mov_b32 v15, v14 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v14, v13 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v13, v12 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v12, v11 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v11, v10 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v10, v9 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v9, v8 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[5:8], off, s33 offset:672 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v20, v16 :: v_dual_mov_b32 v19, v15 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v18, v14 :: v_dual_mov_b32 v17, v13 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v16, v12 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v15, v11 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v14, v10 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v13, v9 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v12, v8 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v11, v7 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v10, v6 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v9, v5 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[5:8], off, s33 offset:656 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x7 ; 128-byte Folded Spill +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[5:8], s33 offset:1744 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[9:12], s33 offset:1760 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[13:16], s33 offset:1776 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[17:20], s33 offset:1792 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[21:24], s33 offset:1808 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[25:28], s33 offset:1824 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[29:32], s33 offset:1840 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[33:36], s33 offset:1856 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x1 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[5:8], off, s33 offset:528 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[57:60], off, s33 offset:608 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(1) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[5:8], s33 offset:1728 ; 16-byte Folded Spill +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[5:8], off, s33 offset:544 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[5:8], s33 offset:1712 ; 16-byte Folded Spill +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[5:8], off, s33 offset:560 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[5:8], s33 offset:1696 ; 16-byte Folded Spill +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[5:8], off, s33 offset:576 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[5:8], s33 offset:1680 ; 16-byte Folded Spill +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[5:8], off, s33 offset:592 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[5:8], s33 offset:1664 ; 16-byte Folded Spill +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[28:31], s32 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x2 ; 44-byte Folded Spill +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[25:28], s33 offset:1872 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[29:32], s33 offset:1888 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b96 off, v[33:35], s33 offset:1904 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x7 ; 128-byte Folded Reload +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[5:8], off, s33 offset:1536 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[9:12], off, s33 offset:1552 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[13:16], off, s33 offset:1568 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[17:20], off, s33 offset:1584 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[21:24], off, s33 offset:1600 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[25:28], off, s33 offset:1616 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[29:32], off, s33 offset:1632 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[33:36], off, s33 offset:1648 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(3) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v24, s2 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x7 ; 128-byte Folded Reload +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[5:8], off, s33 offset:1536 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[9:12], off, s33 offset:1552 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[13:16], off, s33 offset:1568 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[17:20], off, s33 offset:1584 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[21:24], off, s33 offset:1600 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[25:28], off, s33 offset:1616 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[29:32], off, s33 offset:1632 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[33:36], off, s33 offset:1648 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x90 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(3) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[20:23], s2 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x7 ; 128-byte Folded Reload +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[5:8], off, s33 offset:1536 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[9:12], off, s33 offset:1552 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[13:16], off, s33 offset:1568 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[17:20], off, s33 offset:1584 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[21:24], off, s33 offset:1600 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[25:28], off, s33 offset:1616 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[29:32], off, s33 offset:1632 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[33:36], off, s33 offset:1648 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x80 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(4) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[16:19], s2 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x7 ; 128-byte Folded Reload +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[5:8], off, s33 offset:1536 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[9:12], off, s33 offset:1552 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[13:16], off, s33 offset:1568 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[17:20], off, s33 offset:1584 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[21:24], off, s33 offset:1600 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[25:28], off, s33 offset:1616 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[29:32], off, s33 offset:1632 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[33:36], off, s33 offset:1648 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x70 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(5) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[12:15], s2 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x7 ; 128-byte Folded Reload +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[5:8], off, s33 offset:1536 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[9:12], off, s33 offset:1552 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[13:16], off, s33 offset:1568 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[17:20], off, s33 offset:1584 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[21:24], off, s33 offset:1600 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[25:28], off, s33 offset:1616 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[29:32], off, s33 offset:1632 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[33:36], off, s33 offset:1648 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x6c +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b32 off, v0, s2 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x60 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(6) +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b96 off, v[8:10], s2 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x17 ; 384-byte Folded Reload +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[7:10], off, s33 offset:1744 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[11:14], off, s33 offset:1760 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[15:18], off, s33 offset:1776 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[19:22], off, s33 offset:1792 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[23:26], off, s33 offset:1808 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[27:30], off, s33 offset:1824 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[31:34], off, s33 offset:1840 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[35:38], off, s33 offset:1856 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[23:26], off, s33 offset:1536 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[27:30], off, s33 offset:1552 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[31:34], off, s33 offset:1568 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[35:38], off, s33 offset:1584 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[39:42], off, s33 offset:1600 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[43:46], off, s33 offset:1616 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[47:50], off, s33 offset:1632 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[51:54], off, s33 offset:1648 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[24:27], off, s33 offset:1536 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[28:31], off, s33 offset:1552 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[32:35], off, s33 offset:1568 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[36:39], off, s33 offset:1584 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[40:43], off, s33 offset:1600 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[44:47], off, s33 offset:1616 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[48:51], off, s33 offset:1632 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[52:55], off, s33 offset:1648 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 0x50 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(7) +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v24, v25 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x7 ; 128-byte Folded Reload +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[25:28], off, s33 offset:1536 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[29:32], off, s33 offset:1552 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[33:36], off, s33 offset:1568 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[37:40], off, s33 offset:1584 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[41:44], off, s33 offset:1600 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[45:48], off, s33 offset:1616 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[49:52], off, s33 offset:1632 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[53:56], off, s33 offset:1648 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v5, v7 :: v_dual_mov_b32 v6, v8 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(7) +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v7, v9 :: v_dual_mov_b32 v26, v58 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v8, v10 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v9, v11 :: v_dual_mov_b32 v28, v60 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v10, v12 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v11, v13 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(4) +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v12, v14 :: v_dual_mov_b32 v37, v5 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v38, v6 :: v_dual_mov_b32 v39, v7 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v25, v27 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v27, v59 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v13, v15 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v14, v16 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v15, v17 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v16, v18 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v17, v19 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v18, v20 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v19, v21 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v20, v22 :: v_dual_mov_b32 v21, v23 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v22, v24 :: v_dual_mov_b32 v23, v25 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v25, v57 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[20:23], s2 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 64 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[16:19], s2 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 48 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[12:15], s2 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 32 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[8:11], s2 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x2 ; 44-byte Folded Reload +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[8:11], off, s33 offset:1872 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[12:15], off, s33 offset:1888 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b96 v[16:18], off, s33 offset:1904 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s2, s32, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v31, v10 :: v_dual_mov_b32 v36, v15 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v30, v9 :: v_dual_mov_b32 v29, v8 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_store_b128 off, v[36:39], s2 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x4 ; 80-byte Folded Reload +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[5:8], off, s33 offset:1728 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[9:12], off, s33 offset:1712 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[13:16], off, s33 offset:1696 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[17:20], off, s33 offset:1680 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b128 v[21:24], off, s33 offset:1664 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s2, s33, 0x400 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s2 +; GFX11-PAL-GCNTRACKERS-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0xc ; 52-byte Folded Reload +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v60, off, s33 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v59, off, s33 offset:4 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v58, off, s33 offset:8 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v57, off, s33 offset:12 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v56, off, s33 offset:16 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v47, off, s33 offset:20 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v46, off, s33 offset:24 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v45, off, s33 offset:28 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v44, off, s33 offset:32 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v43, off, s33 offset:36 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v42, off, s33 offset:40 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v41, off, s33 offset:44 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v40, off, s33 offset:48 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s30, v61, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s31, v61, 1 +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 s32, s34 +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 s34, s36 +; GFX11-PAL-GCNTRACKERS-NEXT: s_or_saveexec_b32 s0, -1 +; GFX11-PAL-GCNTRACKERS-NEXT: scratch_load_b32 v61, off, s33 offset:1916 ; 4-byte Folded Reload +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 exec_lo, s0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 s33, s35 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31] +; +; TONGA-LABEL: call_72xi32: +; TONGA: ; %bb.0: ; %entry +; TONGA-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; TONGA-NEXT: s_mov_b32 s35, s33 +; TONGA-NEXT: s_add_i32 s33, s32, 0x7fc0 +; TONGA-NEXT: s_and_b32 s33, s33, 0xffff8000 +; TONGA-NEXT: s_or_saveexec_b64 s[36:37], -1 +; TONGA-NEXT: buffer_store_dword v63, off, s[0:3], s33 offset:1564 ; 4-byte Folded Spill +; TONGA-NEXT: s_mov_b64 exec, s[36:37] +; TONGA-NEXT: s_mov_b32 s38, s34 +; TONGA-NEXT: s_mov_b32 s34, s32 +; TONGA-NEXT: s_add_i32 s32, s32, 0x28000 +; TONGA-NEXT: buffer_store_dword v40, off, s[0:3], s33 offset:56 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_store_dword v41, off, s[0:3], s33 offset:52 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_store_dword v42, off, s[0:3], s33 offset:48 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_store_dword v43, off, s[0:3], s33 offset:44 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_store_dword v44, off, s[0:3], s33 offset:40 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_store_dword v45, off, s[0:3], s33 offset:36 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_store_dword v46, off, s[0:3], s33 offset:32 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_store_dword v47, off, s[0:3], s33 offset:28 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_store_dword v56, off, s[0:3], s33 offset:24 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_store_dword v57, off, s[0:3], s33 offset:20 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_store_dword v58, off, s[0:3], s33 offset:16 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_store_dword v59, off, s[0:3], s33 offset:12 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_store_dword v60, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_store_dword v61, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_store_dword v62, off, s[0:3], s33 ; 4-byte Folded Spill +; TONGA-NEXT: v_writelane_b32 v63, s30, 0 +; TONGA-NEXT: v_writelane_b32 v63, s31, 1 +; TONGA-NEXT: s_getpc_b64 s[36:37] +; TONGA-NEXT: s_add_u32 s36, s36, return_72xi32@gotpcrel32@lo+4 +; TONGA-NEXT: s_addc_u32 s37, s37, return_72xi32@gotpcrel32@hi+12 +; TONGA-NEXT: s_load_dwordx2 s[36:37], s[36:37], 0x0 +; TONGA-NEXT: v_mov_b32_e32 v0, 0 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:8 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:12 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:16 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:20 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:28 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:32 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:36 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:40 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:44 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:48 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:52 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 +; TONGA-NEXT: v_lshrrev_b32_e64 v0, 6, s33 +; TONGA-NEXT: s_movk_i32 vcc_lo, 0x200 +; TONGA-NEXT: v_add_u32_e32 v0, vcc, vcc_lo, v0 +; TONGA-NEXT: v_mov_b32_e32 v1, 0 +; TONGA-NEXT: v_mov_b32_e32 v2, 0 +; TONGA-NEXT: v_mov_b32_e32 v3, 0 +; TONGA-NEXT: v_mov_b32_e32 v4, 0 +; TONGA-NEXT: v_mov_b32_e32 v5, 0 +; TONGA-NEXT: v_mov_b32_e32 v6, 0 +; TONGA-NEXT: v_mov_b32_e32 v7, 0 +; TONGA-NEXT: v_mov_b32_e32 v8, 0 +; TONGA-NEXT: v_mov_b32_e32 v9, 0 +; TONGA-NEXT: v_mov_b32_e32 v10, 0 +; TONGA-NEXT: v_mov_b32_e32 v11, 0 +; TONGA-NEXT: v_mov_b32_e32 v12, 0 +; TONGA-NEXT: v_mov_b32_e32 v13, 0 +; TONGA-NEXT: v_mov_b32_e32 v14, 0 +; TONGA-NEXT: v_mov_b32_e32 v15, 0 +; TONGA-NEXT: v_mov_b32_e32 v16, 0 +; TONGA-NEXT: v_mov_b32_e32 v17, 0 +; TONGA-NEXT: v_mov_b32_e32 v18, 0 +; TONGA-NEXT: v_mov_b32_e32 v19, 0 +; TONGA-NEXT: v_mov_b32_e32 v20, 0 +; TONGA-NEXT: v_mov_b32_e32 v21, 0 +; TONGA-NEXT: v_mov_b32_e32 v22, 0 +; TONGA-NEXT: v_mov_b32_e32 v23, 0 +; TONGA-NEXT: v_mov_b32_e32 v24, 0 +; TONGA-NEXT: v_mov_b32_e32 v25, 0 +; TONGA-NEXT: v_mov_b32_e32 v26, 0 +; TONGA-NEXT: v_mov_b32_e32 v27, 0 +; TONGA-NEXT: v_mov_b32_e32 v28, 0 +; TONGA-NEXT: v_mov_b32_e32 v29, 0 +; TONGA-NEXT: v_mov_b32_e32 v30, 0 +; TONGA-NEXT: v_mov_b32_e32 v31, 0 +; TONGA-NEXT: s_waitcnt lgkmcnt(0) +; TONGA-NEXT: s_swappc_b64 s[30:31], s[36:37] +; TONGA-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:636 +; TONGA-NEXT: buffer_load_dword v1, off, s[0:3], s33 offset:640 +; TONGA-NEXT: buffer_load_dword v32, off, s[0:3], s33 offset:644 +; TONGA-NEXT: buffer_load_dword v33, off, s[0:3], s33 offset:648 +; TONGA-NEXT: buffer_load_dword v34, off, s[0:3], s33 offset:652 +; TONGA-NEXT: buffer_load_dword v35, off, s[0:3], s33 offset:656 +; TONGA-NEXT: buffer_load_dword v36, off, s[0:3], s33 offset:660 +; TONGA-NEXT: buffer_load_dword v37, off, s[0:3], s33 offset:664 +; TONGA-NEXT: buffer_load_dword v38, off, s[0:3], s33 offset:668 +; TONGA-NEXT: buffer_load_dword v39, off, s[0:3], s33 offset:672 +; TONGA-NEXT: buffer_load_dword v48, off, s[0:3], s33 offset:676 +; TONGA-NEXT: buffer_load_dword v49, off, s[0:3], s33 offset:680 +; TONGA-NEXT: buffer_load_dword v50, off, s[0:3], s33 offset:684 +; TONGA-NEXT: buffer_load_dword v51, off, s[0:3], s33 offset:688 +; TONGA-NEXT: buffer_load_dword v52, off, s[0:3], s33 offset:692 +; TONGA-NEXT: buffer_load_dword v53, off, s[0:3], s33 offset:696 +; TONGA-NEXT: buffer_load_dword v54, off, s[0:3], s33 offset:700 +; TONGA-NEXT: buffer_load_dword v55, off, s[0:3], s33 offset:704 +; TONGA-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:708 +; TONGA-NEXT: buffer_load_dword v41, off, s[0:3], s33 offset:712 +; TONGA-NEXT: buffer_load_dword v42, off, s[0:3], s33 offset:716 +; TONGA-NEXT: buffer_load_dword v43, off, s[0:3], s33 offset:720 +; TONGA-NEXT: buffer_load_dword v44, off, s[0:3], s33 offset:724 +; TONGA-NEXT: buffer_load_dword v45, off, s[0:3], s33 offset:728 +; TONGA-NEXT: buffer_load_dword v46, off, s[0:3], s33 offset:732 +; TONGA-NEXT: buffer_load_dword v47, off, s[0:3], s33 offset:736 +; TONGA-NEXT: buffer_load_dword v56, off, s[0:3], s33 offset:740 +; TONGA-NEXT: buffer_load_dword v57, off, s[0:3], s33 offset:748 +; TONGA-NEXT: buffer_load_dword v58, off, s[0:3], s33 offset:752 +; TONGA-NEXT: buffer_load_dword v59, off, s[0:3], s33 offset:756 +; TONGA-NEXT: buffer_load_dword v60, off, s[0:3], s33 offset:760 +; TONGA-NEXT: buffer_load_dword v61, off, s[0:3], s33 offset:764 +; TONGA-NEXT: buffer_load_dword v62, off, s[0:3], s33 offset:768 +; TONGA-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:772 +; TONGA-NEXT: buffer_load_dword v3, off, s[0:3], s33 offset:776 +; TONGA-NEXT: buffer_load_dword v4, off, s[0:3], s33 offset:780 +; TONGA-NEXT: buffer_load_dword v5, off, s[0:3], s33 offset:784 +; TONGA-NEXT: buffer_load_dword v6, off, s[0:3], s33 offset:788 +; TONGA-NEXT: buffer_load_dword v7, off, s[0:3], s33 offset:792 +; TONGA-NEXT: buffer_load_dword v8, off, s[0:3], s33 offset:796 +; TONGA-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:516 +; TONGA-NEXT: s_movk_i32 vcc_lo, 0x400 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v9, off, s[0:3], s33 offset:1536 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:520 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v9, off, s[0:3], s33 offset:1540 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:524 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v9, off, s[0:3], s33 offset:1544 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:528 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v9, off, s[0:3], s33 offset:1548 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:532 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v9, off, s[0:3], s33 offset:1552 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:536 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v9, off, s[0:3], s33 offset:1556 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:540 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: buffer_store_dword v9, off, s[0:3], s33 offset:1560 ; 4-byte Folded Spill +; TONGA-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:544 +; TONGA-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:548 +; TONGA-NEXT: buffer_load_dword v11, off, s[0:3], s33 offset:552 +; TONGA-NEXT: buffer_load_dword v12, off, s[0:3], s33 offset:556 +; TONGA-NEXT: buffer_load_dword v13, off, s[0:3], s33 offset:560 +; TONGA-NEXT: buffer_load_dword v14, off, s[0:3], s33 offset:564 +; TONGA-NEXT: buffer_load_dword v15, off, s[0:3], s33 offset:568 +; TONGA-NEXT: buffer_load_dword v16, off, s[0:3], s33 offset:572 +; TONGA-NEXT: buffer_load_dword v17, off, s[0:3], s33 offset:576 +; TONGA-NEXT: buffer_load_dword v18, off, s[0:3], s33 offset:580 +; TONGA-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:584 +; TONGA-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:588 +; TONGA-NEXT: buffer_load_dword v21, off, s[0:3], s33 offset:592 +; TONGA-NEXT: buffer_load_dword v22, off, s[0:3], s33 offset:596 +; TONGA-NEXT: buffer_load_dword v23, off, s[0:3], s33 offset:600 +; TONGA-NEXT: buffer_load_dword v24, off, s[0:3], s33 offset:604 +; TONGA-NEXT: buffer_load_dword v25, off, s[0:3], s33 offset:608 +; TONGA-NEXT: buffer_load_dword v26, off, s[0:3], s33 offset:612 +; TONGA-NEXT: buffer_load_dword v27, off, s[0:3], s33 offset:616 +; TONGA-NEXT: buffer_load_dword v28, off, s[0:3], s33 offset:620 +; TONGA-NEXT: buffer_load_dword v29, off, s[0:3], s33 offset:624 +; TONGA-NEXT: buffer_load_dword v30, off, s[0:3], s33 offset:628 +; TONGA-NEXT: buffer_load_dword v31, off, s[0:3], s33 offset:632 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; TONGA-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 +; TONGA-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:8 +; TONGA-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:12 +; TONGA-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:16 +; TONGA-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:20 +; TONGA-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:24 +; TONGA-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:28 +; TONGA-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:32 +; TONGA-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:36 +; TONGA-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:40 +; TONGA-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:44 +; TONGA-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:48 +; TONGA-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:52 +; TONGA-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:56 +; TONGA-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:60 +; TONGA-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:64 +; TONGA-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:68 +; TONGA-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:72 +; TONGA-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:76 +; TONGA-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:80 +; TONGA-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:84 +; TONGA-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:88 +; TONGA-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:92 +; TONGA-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:96 +; TONGA-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:100 +; TONGA-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:104 +; TONGA-NEXT: v_mov_b32_e32 v0, 24 +; TONGA-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 +; TONGA-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:112 +; TONGA-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:116 +; TONGA-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:120 +; TONGA-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:124 +; TONGA-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:128 +; TONGA-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:132 +; TONGA-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:136 +; TONGA-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:140 +; TONGA-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:144 +; TONGA-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:148 +; TONGA-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:152 +; TONGA-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:156 +; TONGA-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:160 +; TONGA-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:1536 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v3, off, s[0:3], s33 offset:1540 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v4, off, s[0:3], s33 offset:1544 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v5, off, s[0:3], s33 offset:1548 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v6, off, s[0:3], s33 offset:1552 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v7, off, s[0:3], s33 offset:1556 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v8, off, s[0:3], s33 offset:1560 ; 4-byte Folded Reload +; TONGA-NEXT: v_lshrrev_b32_e64 v0, 6, s33 +; TONGA-NEXT: v_add_u32_e32 v0, vcc, vcc_lo, v0 +; TONGA-NEXT: v_mov_b32_e32 v1, 42 +; TONGA-NEXT: s_swappc_b64 s[30:31], s[36:37] +; TONGA-NEXT: buffer_load_dword v62, off, s[0:3], s33 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v61, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v60, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v59, off, s[0:3], s33 offset:12 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v58, off, s[0:3], s33 offset:16 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v57, off, s[0:3], s33 offset:20 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v56, off, s[0:3], s33 offset:24 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v47, off, s[0:3], s33 offset:28 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v46, off, s[0:3], s33 offset:32 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v45, off, s[0:3], s33 offset:36 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v44, off, s[0:3], s33 offset:40 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v43, off, s[0:3], s33 offset:44 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v42, off, s[0:3], s33 offset:48 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v41, off, s[0:3], s33 offset:52 ; 4-byte Folded Reload +; TONGA-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:56 ; 4-byte Folded Reload +; TONGA-NEXT: v_readlane_b32 s30, v63, 0 +; TONGA-NEXT: v_readlane_b32 s31, v63, 1 +; TONGA-NEXT: s_mov_b32 s32, s34 +; TONGA-NEXT: s_mov_b32 s34, s38 +; TONGA-NEXT: s_or_saveexec_b64 s[36:37], -1 +; TONGA-NEXT: buffer_load_dword v63, off, s[0:3], s33 offset:1564 ; 4-byte Folded Reload +; TONGA-NEXT: s_mov_b64 exec, s[36:37] +; TONGA-NEXT: s_mov_b32 s33, s35 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: s_setpc_b64 s[30:31] +; +; TONGA-GCNTRACKERS-LABEL: call_72xi32: +; TONGA-GCNTRACKERS: ; %bb.0: ; %entry +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; TONGA-GCNTRACKERS-NEXT: s_mov_b32 s35, s33 +; TONGA-GCNTRACKERS-NEXT: s_add_i32 s33, s32, 0x7fc0 +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s33, s33, 0xffff8000 +; TONGA-GCNTRACKERS-NEXT: s_or_saveexec_b64 s[36:37], -1 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v63, off, s[0:3], s33 offset:1564 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: s_mov_b64 exec, s[36:37] +; TONGA-GCNTRACKERS-NEXT: s_mov_b32 s38, s34 +; TONGA-GCNTRACKERS-NEXT: s_mov_b32 s34, s32 +; TONGA-GCNTRACKERS-NEXT: s_add_i32 s32, s32, 0x28000 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v40, off, s[0:3], s33 offset:56 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v41, off, s[0:3], s33 offset:52 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v42, off, s[0:3], s33 offset:48 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v43, off, s[0:3], s33 offset:44 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v44, off, s[0:3], s33 offset:40 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v45, off, s[0:3], s33 offset:36 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v46, off, s[0:3], s33 offset:32 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v47, off, s[0:3], s33 offset:28 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v56, off, s[0:3], s33 offset:24 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v57, off, s[0:3], s33 offset:20 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v58, off, s[0:3], s33 offset:16 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v59, off, s[0:3], s33 offset:12 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v60, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v61, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v62, off, s[0:3], s33 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v63, s30, 0 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v63, s31, 1 +; TONGA-GCNTRACKERS-NEXT: s_getpc_b64 s[36:37] +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s36, s36, return_72xi32@gotpcrel32@lo+4 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s37, s37, return_72xi32@gotpcrel32@hi+12 +; TONGA-GCNTRACKERS-NEXT: s_load_dwordx2 s[36:37], s[36:37], 0x0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, 0 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:8 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:12 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:16 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:20 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:28 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:32 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:36 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:40 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:44 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:48 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:52 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 +; TONGA-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 6, s33 +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 vcc_lo, 0x200 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v0, vcc, vcc_lo, v0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v6, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v7, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v8, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v9, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v10, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v11, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v12, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v13, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v14, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v15, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v16, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v17, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v18, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v19, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v20, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v21, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v22, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v23, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v24, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v25, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v26, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v27, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v28, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v29, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v30, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v31, 0 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; TONGA-GCNTRACKERS-NEXT: s_swappc_b64 s[30:31], s[36:37] +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:636 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v1, off, s[0:3], s33 offset:640 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s33 offset:644 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v33, off, s[0:3], s33 offset:648 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v34, off, s[0:3], s33 offset:652 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v35, off, s[0:3], s33 offset:656 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v36, off, s[0:3], s33 offset:660 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v37, off, s[0:3], s33 offset:664 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v38, off, s[0:3], s33 offset:668 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v39, off, s[0:3], s33 offset:672 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v48, off, s[0:3], s33 offset:676 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v49, off, s[0:3], s33 offset:680 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v50, off, s[0:3], s33 offset:684 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v51, off, s[0:3], s33 offset:688 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v52, off, s[0:3], s33 offset:692 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v53, off, s[0:3], s33 offset:696 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v54, off, s[0:3], s33 offset:700 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v55, off, s[0:3], s33 offset:704 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:708 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v41, off, s[0:3], s33 offset:712 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v42, off, s[0:3], s33 offset:716 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v43, off, s[0:3], s33 offset:720 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v44, off, s[0:3], s33 offset:724 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v45, off, s[0:3], s33 offset:728 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v46, off, s[0:3], s33 offset:732 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v47, off, s[0:3], s33 offset:736 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v56, off, s[0:3], s33 offset:740 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v57, off, s[0:3], s33 offset:748 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v58, off, s[0:3], s33 offset:752 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v59, off, s[0:3], s33 offset:756 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v60, off, s[0:3], s33 offset:760 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v61, off, s[0:3], s33 offset:764 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v62, off, s[0:3], s33 offset:768 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:772 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v3, off, s[0:3], s33 offset:776 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v4, off, s[0:3], s33 offset:780 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v5, off, s[0:3], s33 offset:784 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v6, off, s[0:3], s33 offset:788 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v7, off, s[0:3], s33 offset:792 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v8, off, s[0:3], s33 offset:796 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v12, off, s[0:3], s33 offset:516 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v13, off, s[0:3], s33 offset:520 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v14, off, s[0:3], s33 offset:524 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v15, off, s[0:3], s33 offset:528 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v16, off, s[0:3], s33 offset:532 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v17, off, s[0:3], s33 offset:536 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v18, off, s[0:3], s33 offset:540 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:544 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:548 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v11, off, s[0:3], s33 offset:552 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:556 +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 vcc_lo, 0x400 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v19, off, s[0:3], s33 offset:1540 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:560 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v19, off, s[0:3], s33 offset:1544 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:564 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v19, off, s[0:3], s33 offset:1548 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:568 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v19, off, s[0:3], s33 offset:1552 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:572 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v19, off, s[0:3], s33 offset:1556 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:576 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v19, off, s[0:3], s33 offset:1560 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:580 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v19, off, s[0:3], s33 offset:1536 ; 4-byte Folded Spill +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:584 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:588 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v21, off, s[0:3], s33 offset:592 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v22, off, s[0:3], s33 offset:596 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s33 offset:600 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v24, off, s[0:3], s33 offset:604 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v25, off, s[0:3], s33 offset:608 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v26, off, s[0:3], s33 offset:612 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v27, off, s[0:3], s33 offset:616 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v28, off, s[0:3], s33 offset:620 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v29, off, s[0:3], s33 offset:624 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v30, off, s[0:3], s33 offset:628 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v31, off, s[0:3], s33 offset:632 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:8 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:12 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:16 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:20 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:24 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:28 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:32 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:36 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:40 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:44 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:48 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:52 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:56 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:60 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:64 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:68 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:72 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:76 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:80 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:84 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:88 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:92 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:96 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:100 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:104 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, 24 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:112 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:116 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:120 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:124 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:128 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:132 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:136 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:140 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:144 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:148 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:152 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:156 +; TONGA-GCNTRACKERS-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:160 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, v12 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, v13 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, v14 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, v15 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v6, v16 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v7, v17 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v8, v18 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v12, off, s[0:3], s33 offset:1540 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v13, off, s[0:3], s33 offset:1544 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v14, off, s[0:3], s33 offset:1548 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v15, off, s[0:3], s33 offset:1552 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v16, off, s[0:3], s33 offset:1556 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v17, off, s[0:3], s33 offset:1560 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v18, off, s[0:3], s33 offset:1536 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 6, s33 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v0, vcc, vcc_lo, v0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, 42 +; TONGA-GCNTRACKERS-NEXT: s_swappc_b64 s[30:31], s[36:37] +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v62, off, s[0:3], s33 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v61, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v60, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v59, off, s[0:3], s33 offset:12 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v58, off, s[0:3], s33 offset:16 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v57, off, s[0:3], s33 offset:20 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v56, off, s[0:3], s33 offset:24 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v47, off, s[0:3], s33 offset:28 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v46, off, s[0:3], s33 offset:32 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v45, off, s[0:3], s33 offset:36 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v44, off, s[0:3], s33 offset:40 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v43, off, s[0:3], s33 offset:44 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v42, off, s[0:3], s33 offset:48 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v41, off, s[0:3], s33 offset:52 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:56 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s30, v63, 0 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s31, v63, 1 +; TONGA-GCNTRACKERS-NEXT: s_mov_b32 s32, s34 +; TONGA-GCNTRACKERS-NEXT: s_mov_b32 s34, s38 +; TONGA-GCNTRACKERS-NEXT: s_or_saveexec_b64 s[36:37], -1 +; TONGA-GCNTRACKERS-NEXT: buffer_load_dword v63, off, s[0:3], s33 offset:1564 ; 4-byte Folded Reload +; TONGA-GCNTRACKERS-NEXT: s_mov_b64 exec, s[36:37] +; TONGA-GCNTRACKERS-NEXT: s_mov_b32 s33, s35 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31] +; +; GFX908-LABEL: call_72xi32: +; GFX908: ; %bb.0: ; %entry +; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX908-NEXT: s_mov_b32 s35, s33 +; GFX908-NEXT: s_add_i32 s33, s32, 0x7fc0 +; GFX908-NEXT: s_and_b32 s33, s33, 0xffff8000 +; GFX908-NEXT: s_or_saveexec_b64 s[36:37], -1 +; GFX908-NEXT: buffer_store_dword v62, off, s[0:3], s33 offset:1024 ; 4-byte Folded Spill +; GFX908-NEXT: s_mov_b64 exec, s[36:37] +; GFX908-NEXT: s_mov_b32 s38, s34 +; GFX908-NEXT: s_mov_b32 s34, s32 +; GFX908-NEXT: s_add_i32 s32, s32, 0x20000 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v40 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_write_b32 a9, v41 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_write_b32 a10, v42 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_write_b32 a11, v43 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_write_b32 a12, v44 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_write_b32 a13, v45 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_write_b32 a14, v46 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_write_b32 a15, v47 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_write_b32 a16, v56 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_write_b32 a17, v57 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_write_b32 a18, v58 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_write_b32 a19, v59 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_write_b32 a20, v60 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_write_b32 a21, v61 ; Reload Reuse +; GFX908-NEXT: v_writelane_b32 v62, s30, 0 +; GFX908-NEXT: v_writelane_b32 v62, s31, 1 +; GFX908-NEXT: s_getpc_b64 s[36:37] +; GFX908-NEXT: s_add_u32 s36, s36, return_72xi32@gotpcrel32@lo+4 +; GFX908-NEXT: s_addc_u32 s37, s37, return_72xi32@gotpcrel32@hi+12 +; GFX908-NEXT: s_load_dwordx2 s[36:37], s[36:37], 0x0 +; GFX908-NEXT: v_mov_b32_e32 v0, 0 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:8 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:12 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:16 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:20 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:28 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:32 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:36 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:40 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:44 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:48 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:52 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 +; GFX908-NEXT: v_lshrrev_b32_e64 v0, 6, s33 +; GFX908-NEXT: v_mov_b32_e32 v1, 0 +; GFX908-NEXT: v_mov_b32_e32 v2, 0 +; GFX908-NEXT: v_mov_b32_e32 v3, 0 +; GFX908-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-NEXT: v_mov_b32_e32 v5, 0 +; GFX908-NEXT: v_mov_b32_e32 v6, 0 +; GFX908-NEXT: v_mov_b32_e32 v7, 0 +; GFX908-NEXT: v_mov_b32_e32 v8, 0 +; GFX908-NEXT: v_mov_b32_e32 v9, 0 +; GFX908-NEXT: v_mov_b32_e32 v10, 0 +; GFX908-NEXT: v_mov_b32_e32 v11, 0 +; GFX908-NEXT: v_mov_b32_e32 v12, 0 +; GFX908-NEXT: v_mov_b32_e32 v13, 0 +; GFX908-NEXT: v_mov_b32_e32 v14, 0 +; GFX908-NEXT: v_mov_b32_e32 v15, 0 +; GFX908-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-NEXT: v_mov_b32_e32 v17, 0 +; GFX908-NEXT: v_mov_b32_e32 v18, 0 +; GFX908-NEXT: v_mov_b32_e32 v19, 0 +; GFX908-NEXT: v_mov_b32_e32 v20, 0 +; GFX908-NEXT: v_mov_b32_e32 v21, 0 +; GFX908-NEXT: v_mov_b32_e32 v22, 0 +; GFX908-NEXT: v_mov_b32_e32 v23, 0 +; GFX908-NEXT: v_mov_b32_e32 v24, 0 +; GFX908-NEXT: v_mov_b32_e32 v25, 0 +; GFX908-NEXT: v_mov_b32_e32 v26, 0 +; GFX908-NEXT: v_mov_b32_e32 v27, 0 +; GFX908-NEXT: v_mov_b32_e32 v28, 0 +; GFX908-NEXT: v_mov_b32_e32 v29, 0 +; GFX908-NEXT: v_mov_b32_e32 v30, 0 +; GFX908-NEXT: v_mov_b32_e32 v31, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX908-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:124 +; GFX908-NEXT: buffer_load_dword v1, off, s[0:3], s33 offset:128 +; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s33 offset:132 +; GFX908-NEXT: buffer_load_dword v33, off, s[0:3], s33 offset:136 +; GFX908-NEXT: buffer_load_dword v34, off, s[0:3], s33 offset:140 +; GFX908-NEXT: buffer_load_dword v35, off, s[0:3], s33 offset:144 +; GFX908-NEXT: buffer_load_dword v36, off, s[0:3], s33 offset:148 +; GFX908-NEXT: buffer_load_dword v37, off, s[0:3], s33 offset:152 +; GFX908-NEXT: buffer_load_dword v38, off, s[0:3], s33 offset:156 +; GFX908-NEXT: buffer_load_dword v39, off, s[0:3], s33 offset:160 +; GFX908-NEXT: buffer_load_dword v48, off, s[0:3], s33 offset:164 +; GFX908-NEXT: buffer_load_dword v49, off, s[0:3], s33 offset:168 +; GFX908-NEXT: buffer_load_dword v50, off, s[0:3], s33 offset:172 +; GFX908-NEXT: buffer_load_dword v51, off, s[0:3], s33 offset:176 +; GFX908-NEXT: buffer_load_dword v52, off, s[0:3], s33 offset:180 +; GFX908-NEXT: buffer_load_dword v53, off, s[0:3], s33 offset:184 +; GFX908-NEXT: buffer_load_dword v54, off, s[0:3], s33 offset:188 +; GFX908-NEXT: buffer_load_dword v55, off, s[0:3], s33 offset:192 +; GFX908-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:196 +; GFX908-NEXT: buffer_load_dword v41, off, s[0:3], s33 offset:200 +; GFX908-NEXT: buffer_load_dword v42, off, s[0:3], s33 offset:204 +; GFX908-NEXT: buffer_load_dword v43, off, s[0:3], s33 offset:208 +; GFX908-NEXT: buffer_load_dword v44, off, s[0:3], s33 offset:212 +; GFX908-NEXT: buffer_load_dword v45, off, s[0:3], s33 offset:216 +; GFX908-NEXT: buffer_load_dword v46, off, s[0:3], s33 offset:220 +; GFX908-NEXT: buffer_load_dword v47, off, s[0:3], s33 offset:224 +; GFX908-NEXT: buffer_load_dword v56, off, s[0:3], s33 offset:228 +; GFX908-NEXT: buffer_load_dword v57, off, s[0:3], s33 offset:236 +; GFX908-NEXT: buffer_load_dword v58, off, s[0:3], s33 offset:240 +; GFX908-NEXT: buffer_load_dword v59, off, s[0:3], s33 offset:244 +; GFX908-NEXT: buffer_load_dword v60, off, s[0:3], s33 offset:248 +; GFX908-NEXT: buffer_load_dword v61, off, s[0:3], s33 offset:252 +; GFX908-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:256 +; GFX908-NEXT: buffer_load_dword v3, off, s[0:3], s33 offset:260 +; GFX908-NEXT: buffer_load_dword v4, off, s[0:3], s33 offset:264 +; GFX908-NEXT: buffer_load_dword v5, off, s[0:3], s33 offset:268 +; GFX908-NEXT: buffer_load_dword v6, off, s[0:3], s33 offset:272 +; GFX908-NEXT: buffer_load_dword v7, off, s[0:3], s33 offset:276 +; GFX908-NEXT: buffer_load_dword v8, off, s[0:3], s33 offset:280 +; GFX908-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:284 +; GFX908-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:4 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: v_accvgpr_write_b32 a0, v10 +; GFX908-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:8 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: v_accvgpr_write_b32 a1, v10 +; GFX908-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:12 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: v_accvgpr_write_b32 a2, v10 +; GFX908-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:16 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: v_accvgpr_write_b32 a3, v10 +; GFX908-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:20 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: v_accvgpr_write_b32 a4, v10 +; GFX908-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:24 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: v_accvgpr_write_b32 a5, v10 +; GFX908-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:28 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: v_accvgpr_write_b32 a6, v10 +; GFX908-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:32 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: v_accvgpr_write_b32 a7, v10 +; GFX908-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:36 +; GFX908-NEXT: buffer_load_dword v11, off, s[0:3], s33 offset:40 +; GFX908-NEXT: buffer_load_dword v12, off, s[0:3], s33 offset:44 +; GFX908-NEXT: buffer_load_dword v13, off, s[0:3], s33 offset:48 +; GFX908-NEXT: buffer_load_dword v14, off, s[0:3], s33 offset:52 +; GFX908-NEXT: buffer_load_dword v15, off, s[0:3], s33 offset:56 +; GFX908-NEXT: buffer_load_dword v16, off, s[0:3], s33 offset:60 +; GFX908-NEXT: buffer_load_dword v17, off, s[0:3], s33 offset:64 +; GFX908-NEXT: buffer_load_dword v18, off, s[0:3], s33 offset:68 +; GFX908-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:72 +; GFX908-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:76 +; GFX908-NEXT: buffer_load_dword v21, off, s[0:3], s33 offset:80 +; GFX908-NEXT: buffer_load_dword v22, off, s[0:3], s33 offset:84 +; GFX908-NEXT: buffer_load_dword v23, off, s[0:3], s33 offset:88 +; GFX908-NEXT: buffer_load_dword v24, off, s[0:3], s33 offset:92 +; GFX908-NEXT: buffer_load_dword v25, off, s[0:3], s33 offset:96 +; GFX908-NEXT: buffer_load_dword v26, off, s[0:3], s33 offset:100 +; GFX908-NEXT: buffer_load_dword v27, off, s[0:3], s33 offset:104 +; GFX908-NEXT: buffer_load_dword v28, off, s[0:3], s33 offset:108 +; GFX908-NEXT: buffer_load_dword v29, off, s[0:3], s33 offset:112 +; GFX908-NEXT: buffer_load_dword v30, off, s[0:3], s33 offset:116 +; GFX908-NEXT: buffer_load_dword v31, off, s[0:3], s33 offset:120 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; GFX908-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 +; GFX908-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:8 +; GFX908-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:12 +; GFX908-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:16 +; GFX908-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:20 +; GFX908-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:24 +; GFX908-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:28 +; GFX908-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:32 +; GFX908-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:36 +; GFX908-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:40 +; GFX908-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:44 +; GFX908-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:48 +; GFX908-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:52 +; GFX908-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:56 +; GFX908-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:60 +; GFX908-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:64 +; GFX908-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:68 +; GFX908-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:72 +; GFX908-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:76 +; GFX908-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:80 +; GFX908-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:84 +; GFX908-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:88 +; GFX908-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:92 +; GFX908-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:96 +; GFX908-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:100 +; GFX908-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:104 +; GFX908-NEXT: v_mov_b32_e32 v0, 24 +; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 +; GFX908-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:112 +; GFX908-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:116 +; GFX908-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:120 +; GFX908-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:124 +; GFX908-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:128 +; GFX908-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:132 +; GFX908-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:136 +; GFX908-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:140 +; GFX908-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:144 +; GFX908-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:148 +; GFX908-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:152 +; GFX908-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:156 +; GFX908-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:160 +; GFX908-NEXT: v_lshrrev_b32_e64 v0, 6, s33 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a4 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a7 +; GFX908-NEXT: v_add_u32_e32 v0, 0x200, v0 +; GFX908-NEXT: v_mov_b32_e32 v1, 42 +; GFX908-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX908-NEXT: v_readlane_b32 s30, v62, 0 +; GFX908-NEXT: v_readlane_b32 s31, v62, 1 +; GFX908-NEXT: v_accvgpr_read_b32 v61, a21 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_read_b32 v60, a20 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_read_b32 v59, a19 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_read_b32 v58, a18 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_read_b32 v57, a17 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_read_b32 v56, a16 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_read_b32 v47, a15 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_read_b32 v46, a14 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_read_b32 v45, a13 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_read_b32 v44, a12 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_read_b32 v43, a11 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_read_b32 v42, a10 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_read_b32 v41, a9 ; Reload Reuse +; GFX908-NEXT: v_accvgpr_read_b32 v40, a8 ; Reload Reuse +; GFX908-NEXT: s_mov_b32 s32, s34 +; GFX908-NEXT: s_mov_b32 s34, s38 +; GFX908-NEXT: s_or_saveexec_b64 s[36:37], -1 +; GFX908-NEXT: buffer_load_dword v62, off, s[0:3], s33 offset:1024 ; 4-byte Folded Reload +; GFX908-NEXT: s_mov_b64 exec, s[36:37] +; GFX908-NEXT: s_mov_b32 s33, s35 +; GFX908-NEXT: s_waitcnt vmcnt(0) +; GFX908-NEXT: s_setpc_b64 s[30:31] +; +; GFX908-GCNTRACKERS-LABEL: call_72xi32: +; GFX908-GCNTRACKERS: ; %bb.0: ; %entry +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX908-GCNTRACKERS-NEXT: s_mov_b32 s35, s33 +; GFX908-GCNTRACKERS-NEXT: s_add_i32 s33, s32, 0x7fc0 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s33, s33, 0xffff8000 +; GFX908-GCNTRACKERS-NEXT: s_or_saveexec_b64 s[36:37], -1 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v62, off, s[0:3], s33 offset:1024 ; 4-byte Folded Spill +; GFX908-GCNTRACKERS-NEXT: s_mov_b64 exec, s[36:37] +; GFX908-GCNTRACKERS-NEXT: s_mov_b32 s38, s34 +; GFX908-GCNTRACKERS-NEXT: s_mov_b32 s34, s32 +; GFX908-GCNTRACKERS-NEXT: s_add_i32 s32, s32, 0x20000 +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a8, v40 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a9, v41 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a10, v42 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a11, v43 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a12, v44 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a13, v45 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a14, v46 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a15, v47 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a16, v56 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a17, v57 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a18, v58 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a19, v59 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a20, v60 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a21, v61 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_writelane_b32 v62, s30, 0 +; GFX908-GCNTRACKERS-NEXT: v_writelane_b32 v62, s31, 1 +; GFX908-GCNTRACKERS-NEXT: s_getpc_b64 s[36:37] +; GFX908-GCNTRACKERS-NEXT: s_add_u32 s36, s36, return_72xi32@gotpcrel32@lo+4 +; GFX908-GCNTRACKERS-NEXT: s_addc_u32 s37, s37, return_72xi32@gotpcrel32@hi+12 +; GFX908-GCNTRACKERS-NEXT: s_load_dwordx2 s[36:37], s[36:37], 0x0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, 0 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:8 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:12 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:16 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:20 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:28 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:32 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:36 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:40 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:44 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:48 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:52 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 +; GFX908-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 6, s33 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v6, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v7, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v8, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v9, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v10, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v11, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v12, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v13, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v14, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v15, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v17, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v18, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v19, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v20, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v21, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v22, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v23, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v24, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v25, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v26, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v27, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v28, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v29, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v30, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v31, 0 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GCNTRACKERS-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:124 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v1, off, s[0:3], s33 offset:128 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s33 offset:132 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v33, off, s[0:3], s33 offset:136 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v34, off, s[0:3], s33 offset:140 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v35, off, s[0:3], s33 offset:144 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v36, off, s[0:3], s33 offset:148 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v37, off, s[0:3], s33 offset:152 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v38, off, s[0:3], s33 offset:156 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v39, off, s[0:3], s33 offset:160 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v48, off, s[0:3], s33 offset:164 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v49, off, s[0:3], s33 offset:168 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v50, off, s[0:3], s33 offset:172 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v51, off, s[0:3], s33 offset:176 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v52, off, s[0:3], s33 offset:180 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v53, off, s[0:3], s33 offset:184 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v54, off, s[0:3], s33 offset:188 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v55, off, s[0:3], s33 offset:192 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:196 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v41, off, s[0:3], s33 offset:200 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v42, off, s[0:3], s33 offset:204 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v43, off, s[0:3], s33 offset:208 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v44, off, s[0:3], s33 offset:212 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v45, off, s[0:3], s33 offset:216 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v46, off, s[0:3], s33 offset:220 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v47, off, s[0:3], s33 offset:224 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v56, off, s[0:3], s33 offset:228 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v57, off, s[0:3], s33 offset:236 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v58, off, s[0:3], s33 offset:240 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v59, off, s[0:3], s33 offset:244 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v60, off, s[0:3], s33 offset:248 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v61, off, s[0:3], s33 offset:252 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:256 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v3, off, s[0:3], s33 offset:260 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v4, off, s[0:3], s33 offset:264 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v5, off, s[0:3], s33 offset:268 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v6, off, s[0:3], s33 offset:272 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v7, off, s[0:3], s33 offset:276 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v8, off, s[0:3], s33 offset:280 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:284 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v12, off, s[0:3], s33 offset:4 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v13, off, s[0:3], s33 offset:8 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v14, off, s[0:3], s33 offset:12 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v15, off, s[0:3], s33 offset:16 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v16, off, s[0:3], s33 offset:20 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v17, off, s[0:3], s33 offset:24 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v18, off, s[0:3], s33 offset:28 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:32 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:36 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v11, off, s[0:3], s33 offset:40 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:44 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a2, v20 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:48 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a3, v20 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:52 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a4, v20 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:56 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a5, v20 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:60 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a6, v20 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:64 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a7, v20 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:68 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a1, v20 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:72 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_write_b32 a0, v20 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:76 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v21, off, s[0:3], s33 offset:80 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v22, off, s[0:3], s33 offset:84 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s33 offset:88 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v24, off, s[0:3], s33 offset:92 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v25, off, s[0:3], s33 offset:96 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v26, off, s[0:3], s33 offset:100 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v27, off, s[0:3], s33 offset:104 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v28, off, s[0:3], s33 offset:108 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v29, off, s[0:3], s33 offset:112 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v30, off, s[0:3], s33 offset:116 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v31, off, s[0:3], s33 offset:120 +; GFX908-GCNTRACKERS-NEXT: s_nop 0 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:8 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:12 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:16 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:20 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:24 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:28 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:32 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:36 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:40 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:44 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:48 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:52 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:56 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:60 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:64 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:68 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:72 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:76 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:80 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:84 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:88 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:92 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:96 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:100 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:104 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, 24 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:112 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:116 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:120 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:124 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:128 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:132 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:136 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:140 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:144 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:148 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:152 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:156 +; GFX908-GCNTRACKERS-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:160 +; GFX908-GCNTRACKERS-NEXT: v_lshrrev_b32_e64 v0, 6, s33 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, v12 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, v13 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, v14 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, v15 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v6, v16 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v7, v17 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v8, v18 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v9, v19 +; GFX908-GCNTRACKERS-NEXT: v_add_u32_e32 v0, 0x200, v0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, 42 +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v12, a2 +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v13, a3 +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v14, a4 +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v15, a5 +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v16, a6 +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v17, a7 +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v18, a1 +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v19, a0 +; GFX908-GCNTRACKERS-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX908-GCNTRACKERS-NEXT: v_readlane_b32 s30, v62, 0 +; GFX908-GCNTRACKERS-NEXT: v_readlane_b32 s31, v62, 1 +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v61, a21 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v60, a20 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v59, a19 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v58, a18 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v57, a17 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v56, a16 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v47, a15 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v46, a14 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v45, a13 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v44, a12 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v43, a11 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v42, a10 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v41, a9 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: v_accvgpr_read_b32 v40, a8 ; Reload Reuse +; GFX908-GCNTRACKERS-NEXT: s_mov_b32 s32, s34 +; GFX908-GCNTRACKERS-NEXT: s_mov_b32 s34, s38 +; GFX908-GCNTRACKERS-NEXT: s_or_saveexec_b64 s[36:37], -1 +; GFX908-GCNTRACKERS-NEXT: buffer_load_dword v62, off, s[0:3], s33 offset:1024 ; 4-byte Folded Reload +; GFX908-GCNTRACKERS-NEXT: s_mov_b64 exec, s[36:37] +; GFX908-GCNTRACKERS-NEXT: s_mov_b32 s33, s35 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX908-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31] +; +; GENERIC-LABEL: call_72xi32: +; GENERIC: ; %bb.0: ; %entry +; GENERIC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GENERIC-NEXT: s_mov_b32 s35, s33 +; GENERIC-NEXT: s_add_i32 s33, s32, 0x7fc0 +; GENERIC-NEXT: s_and_b32 s33, s33, 0xffff8000 +; GENERIC-NEXT: s_or_saveexec_b64 s[36:37], -1 +; GENERIC-NEXT: buffer_store_dword v63, off, s[0:3], s33 offset:1564 ; 4-byte Folded Spill +; GENERIC-NEXT: s_mov_b64 exec, s[36:37] +; GENERIC-NEXT: s_mov_b32 s38, s34 +; GENERIC-NEXT: s_mov_b32 s34, s32 +; GENERIC-NEXT: s_add_i32 s32, s32, 0x28000 +; GENERIC-NEXT: buffer_store_dword v40, off, s[0:3], s33 offset:56 ; 4-byte Folded Spill +; GENERIC-NEXT: buffer_store_dword v41, off, s[0:3], s33 offset:52 ; 4-byte Folded Spill +; GENERIC-NEXT: buffer_store_dword v42, off, s[0:3], s33 offset:48 ; 4-byte Folded Spill +; GENERIC-NEXT: buffer_store_dword v43, off, s[0:3], s33 offset:44 ; 4-byte Folded Spill +; GENERIC-NEXT: buffer_store_dword v44, off, s[0:3], s33 offset:40 ; 4-byte Folded Spill +; GENERIC-NEXT: buffer_store_dword v45, off, s[0:3], s33 offset:36 ; 4-byte Folded Spill +; GENERIC-NEXT: buffer_store_dword v46, off, s[0:3], s33 offset:32 ; 4-byte Folded Spill +; GENERIC-NEXT: buffer_store_dword v47, off, s[0:3], s33 offset:28 ; 4-byte Folded Spill +; GENERIC-NEXT: buffer_store_dword v56, off, s[0:3], s33 offset:24 ; 4-byte Folded Spill +; GENERIC-NEXT: buffer_store_dword v57, off, s[0:3], s33 offset:20 ; 4-byte Folded Spill +; GENERIC-NEXT: buffer_store_dword v58, off, s[0:3], s33 offset:16 ; 4-byte Folded Spill +; GENERIC-NEXT: buffer_store_dword v59, off, s[0:3], s33 offset:12 ; 4-byte Folded Spill +; GENERIC-NEXT: buffer_store_dword v60, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill +; GENERIC-NEXT: buffer_store_dword v61, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill +; GENERIC-NEXT: buffer_store_dword v62, off, s[0:3], s33 ; 4-byte Folded Spill +; GENERIC-NEXT: v_writelane_b32 v63, s30, 0 +; GENERIC-NEXT: v_writelane_b32 v63, s31, 1 +; GENERIC-NEXT: s_getpc_b64 s[36:37] +; GENERIC-NEXT: s_add_u32 s36, s36, return_72xi32@gotpcrel32@lo+4 +; GENERIC-NEXT: s_addc_u32 s37, s37, return_72xi32@gotpcrel32@hi+12 +; GENERIC-NEXT: s_load_dwordx2 s[36:37], s[36:37], 0x0 +; GENERIC-NEXT: v_mov_b32_e32 v0, 0 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:8 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:12 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:16 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:20 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:28 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:32 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:36 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:40 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:44 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:48 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:52 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_lshr_b32_e64 v0, s33, 6 +; GENERIC-NEXT: s_movk_i32 vcc_lo, 0x200 +; GENERIC-NEXT: v_add_i32_e32 v0, vcc, vcc_lo, v0 +; GENERIC-NEXT: v_mov_b32_e32 v1, 0 +; GENERIC-NEXT: v_mov_b32_e32 v2, 0 +; GENERIC-NEXT: v_mov_b32_e32 v3, 0 +; GENERIC-NEXT: v_mov_b32_e32 v4, 0 +; GENERIC-NEXT: v_mov_b32_e32 v5, 0 +; GENERIC-NEXT: v_mov_b32_e32 v6, 0 +; GENERIC-NEXT: v_mov_b32_e32 v7, 0 +; GENERIC-NEXT: v_mov_b32_e32 v8, 0 +; GENERIC-NEXT: v_mov_b32_e32 v9, 0 +; GENERIC-NEXT: v_mov_b32_e32 v10, 0 +; GENERIC-NEXT: v_mov_b32_e32 v11, 0 +; GENERIC-NEXT: v_mov_b32_e32 v12, 0 +; GENERIC-NEXT: v_mov_b32_e32 v13, 0 +; GENERIC-NEXT: v_mov_b32_e32 v14, 0 +; GENERIC-NEXT: v_mov_b32_e32 v15, 0 +; GENERIC-NEXT: v_mov_b32_e32 v16, 0 +; GENERIC-NEXT: v_mov_b32_e32 v17, 0 +; GENERIC-NEXT: v_mov_b32_e32 v18, 0 +; GENERIC-NEXT: v_mov_b32_e32 v19, 0 +; GENERIC-NEXT: v_mov_b32_e32 v20, 0 +; GENERIC-NEXT: v_mov_b32_e32 v21, 0 +; GENERIC-NEXT: v_mov_b32_e32 v22, 0 +; GENERIC-NEXT: v_mov_b32_e32 v23, 0 +; GENERIC-NEXT: v_mov_b32_e32 v24, 0 +; GENERIC-NEXT: v_mov_b32_e32 v25, 0 +; GENERIC-NEXT: v_mov_b32_e32 v26, 0 +; GENERIC-NEXT: v_mov_b32_e32 v27, 0 +; GENERIC-NEXT: v_mov_b32_e32 v28, 0 +; GENERIC-NEXT: v_mov_b32_e32 v29, 0 +; GENERIC-NEXT: v_mov_b32_e32 v30, 0 +; GENERIC-NEXT: v_mov_b32_e32 v31, 0 +; GENERIC-NEXT: s_waitcnt lgkmcnt(0) +; GENERIC-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GENERIC-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:636 +; GENERIC-NEXT: buffer_load_dword v1, off, s[0:3], s33 offset:640 +; GENERIC-NEXT: buffer_load_dword v32, off, s[0:3], s33 offset:644 +; GENERIC-NEXT: buffer_load_dword v33, off, s[0:3], s33 offset:648 +; GENERIC-NEXT: buffer_load_dword v34, off, s[0:3], s33 offset:652 +; GENERIC-NEXT: buffer_load_dword v35, off, s[0:3], s33 offset:656 +; GENERIC-NEXT: buffer_load_dword v36, off, s[0:3], s33 offset:660 +; GENERIC-NEXT: buffer_load_dword v37, off, s[0:3], s33 offset:664 +; GENERIC-NEXT: buffer_load_dword v38, off, s[0:3], s33 offset:668 +; GENERIC-NEXT: buffer_load_dword v39, off, s[0:3], s33 offset:672 +; GENERIC-NEXT: buffer_load_dword v48, off, s[0:3], s33 offset:676 +; GENERIC-NEXT: buffer_load_dword v49, off, s[0:3], s33 offset:680 +; GENERIC-NEXT: buffer_load_dword v50, off, s[0:3], s33 offset:684 +; GENERIC-NEXT: buffer_load_dword v51, off, s[0:3], s33 offset:688 +; GENERIC-NEXT: buffer_load_dword v52, off, s[0:3], s33 offset:692 +; GENERIC-NEXT: buffer_load_dword v53, off, s[0:3], s33 offset:696 +; GENERIC-NEXT: buffer_load_dword v54, off, s[0:3], s33 offset:700 +; GENERIC-NEXT: buffer_load_dword v55, off, s[0:3], s33 offset:704 +; GENERIC-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:708 +; GENERIC-NEXT: buffer_load_dword v41, off, s[0:3], s33 offset:712 +; GENERIC-NEXT: buffer_load_dword v42, off, s[0:3], s33 offset:716 +; GENERIC-NEXT: buffer_load_dword v43, off, s[0:3], s33 offset:720 +; GENERIC-NEXT: buffer_load_dword v44, off, s[0:3], s33 offset:724 +; GENERIC-NEXT: buffer_load_dword v45, off, s[0:3], s33 offset:728 +; GENERIC-NEXT: buffer_load_dword v46, off, s[0:3], s33 offset:732 +; GENERIC-NEXT: buffer_load_dword v47, off, s[0:3], s33 offset:736 +; GENERIC-NEXT: buffer_load_dword v56, off, s[0:3], s33 offset:740 +; GENERIC-NEXT: buffer_load_dword v57, off, s[0:3], s33 offset:748 +; GENERIC-NEXT: buffer_load_dword v58, off, s[0:3], s33 offset:752 +; GENERIC-NEXT: buffer_load_dword v59, off, s[0:3], s33 offset:756 +; GENERIC-NEXT: buffer_load_dword v60, off, s[0:3], s33 offset:760 +; GENERIC-NEXT: buffer_load_dword v61, off, s[0:3], s33 offset:764 +; GENERIC-NEXT: buffer_load_dword v62, off, s[0:3], s33 offset:768 +; GENERIC-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:772 +; GENERIC-NEXT: buffer_load_dword v3, off, s[0:3], s33 offset:776 +; GENERIC-NEXT: buffer_load_dword v4, off, s[0:3], s33 offset:780 +; GENERIC-NEXT: buffer_load_dword v5, off, s[0:3], s33 offset:784 +; GENERIC-NEXT: buffer_load_dword v6, off, s[0:3], s33 offset:788 +; GENERIC-NEXT: buffer_load_dword v7, off, s[0:3], s33 offset:792 +; GENERIC-NEXT: buffer_load_dword v8, off, s[0:3], s33 offset:796 +; GENERIC-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:516 +; GENERIC-NEXT: s_movk_i32 vcc_lo, 0x400 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v9, off, s[0:3], s33 offset:1536 ; 4-byte Folded Spill +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:520 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v9, off, s[0:3], s33 offset:1540 ; 4-byte Folded Spill +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:524 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v9, off, s[0:3], s33 offset:1544 ; 4-byte Folded Spill +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:528 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v9, off, s[0:3], s33 offset:1548 ; 4-byte Folded Spill +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:532 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v9, off, s[0:3], s33 offset:1552 ; 4-byte Folded Spill +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:536 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v9, off, s[0:3], s33 offset:1556 ; 4-byte Folded Spill +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:540 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: buffer_store_dword v9, off, s[0:3], s33 offset:1560 ; 4-byte Folded Spill +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:544 +; GENERIC-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:548 +; GENERIC-NEXT: buffer_load_dword v11, off, s[0:3], s33 offset:552 +; GENERIC-NEXT: buffer_load_dword v12, off, s[0:3], s33 offset:556 +; GENERIC-NEXT: buffer_load_dword v13, off, s[0:3], s33 offset:560 +; GENERIC-NEXT: buffer_load_dword v14, off, s[0:3], s33 offset:564 +; GENERIC-NEXT: buffer_load_dword v15, off, s[0:3], s33 offset:568 +; GENERIC-NEXT: buffer_load_dword v16, off, s[0:3], s33 offset:572 +; GENERIC-NEXT: buffer_load_dword v17, off, s[0:3], s33 offset:576 +; GENERIC-NEXT: buffer_load_dword v18, off, s[0:3], s33 offset:580 +; GENERIC-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:584 +; GENERIC-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:588 +; GENERIC-NEXT: buffer_load_dword v21, off, s[0:3], s33 offset:592 +; GENERIC-NEXT: buffer_load_dword v22, off, s[0:3], s33 offset:596 +; GENERIC-NEXT: buffer_load_dword v23, off, s[0:3], s33 offset:600 +; GENERIC-NEXT: buffer_load_dword v24, off, s[0:3], s33 offset:604 +; GENERIC-NEXT: buffer_load_dword v25, off, s[0:3], s33 offset:608 +; GENERIC-NEXT: buffer_load_dword v26, off, s[0:3], s33 offset:612 +; GENERIC-NEXT: buffer_load_dword v27, off, s[0:3], s33 offset:616 +; GENERIC-NEXT: buffer_load_dword v28, off, s[0:3], s33 offset:620 +; GENERIC-NEXT: buffer_load_dword v29, off, s[0:3], s33 offset:624 +; GENERIC-NEXT: buffer_load_dword v30, off, s[0:3], s33 offset:628 +; GENERIC-NEXT: buffer_load_dword v31, off, s[0:3], s33 offset:632 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; GENERIC-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 +; GENERIC-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:8 +; GENERIC-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:12 +; GENERIC-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:16 +; GENERIC-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:20 +; GENERIC-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:24 +; GENERIC-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:28 +; GENERIC-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:32 +; GENERIC-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:36 +; GENERIC-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:40 +; GENERIC-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:44 +; GENERIC-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:48 +; GENERIC-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:52 +; GENERIC-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:56 +; GENERIC-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:60 +; GENERIC-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:64 +; GENERIC-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:68 +; GENERIC-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:72 +; GENERIC-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:76 +; GENERIC-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:80 +; GENERIC-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:84 +; GENERIC-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:88 +; GENERIC-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:92 +; GENERIC-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:96 +; GENERIC-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:100 +; GENERIC-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:104 +; GENERIC-NEXT: v_mov_b32_e32 v0, 24 +; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 +; GENERIC-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:112 +; GENERIC-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:116 +; GENERIC-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:120 +; GENERIC-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:124 +; GENERIC-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:128 +; GENERIC-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:132 +; GENERIC-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:136 +; GENERIC-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:140 +; GENERIC-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:144 +; GENERIC-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:148 +; GENERIC-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:152 +; GENERIC-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:156 +; GENERIC-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:160 +; GENERIC-NEXT: s_waitcnt expcnt(6) +; GENERIC-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:1536 ; 4-byte Folded Reload +; GENERIC-NEXT: s_waitcnt expcnt(5) +; GENERIC-NEXT: buffer_load_dword v3, off, s[0:3], s33 offset:1540 ; 4-byte Folded Reload +; GENERIC-NEXT: s_waitcnt expcnt(4) +; GENERIC-NEXT: buffer_load_dword v4, off, s[0:3], s33 offset:1544 ; 4-byte Folded Reload +; GENERIC-NEXT: s_waitcnt expcnt(3) +; GENERIC-NEXT: buffer_load_dword v5, off, s[0:3], s33 offset:1548 ; 4-byte Folded Reload +; GENERIC-NEXT: s_waitcnt expcnt(2) +; GENERIC-NEXT: buffer_load_dword v6, off, s[0:3], s33 offset:1552 ; 4-byte Folded Reload +; GENERIC-NEXT: s_waitcnt expcnt(1) +; GENERIC-NEXT: buffer_load_dword v7, off, s[0:3], s33 offset:1556 ; 4-byte Folded Reload +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: buffer_load_dword v8, off, s[0:3], s33 offset:1560 ; 4-byte Folded Reload +; GENERIC-NEXT: v_lshr_b32_e64 v0, s33, 6 +; GENERIC-NEXT: v_add_i32_e32 v0, vcc, vcc_lo, v0 +; GENERIC-NEXT: v_mov_b32_e32 v1, 42 +; GENERIC-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GENERIC-NEXT: buffer_load_dword v62, off, s[0:3], s33 ; 4-byte Folded Reload +; GENERIC-NEXT: buffer_load_dword v61, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload +; GENERIC-NEXT: buffer_load_dword v60, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload +; GENERIC-NEXT: buffer_load_dword v59, off, s[0:3], s33 offset:12 ; 4-byte Folded Reload +; GENERIC-NEXT: buffer_load_dword v58, off, s[0:3], s33 offset:16 ; 4-byte Folded Reload +; GENERIC-NEXT: buffer_load_dword v57, off, s[0:3], s33 offset:20 ; 4-byte Folded Reload +; GENERIC-NEXT: buffer_load_dword v56, off, s[0:3], s33 offset:24 ; 4-byte Folded Reload +; GENERIC-NEXT: buffer_load_dword v47, off, s[0:3], s33 offset:28 ; 4-byte Folded Reload +; GENERIC-NEXT: buffer_load_dword v46, off, s[0:3], s33 offset:32 ; 4-byte Folded Reload +; GENERIC-NEXT: buffer_load_dword v45, off, s[0:3], s33 offset:36 ; 4-byte Folded Reload +; GENERIC-NEXT: buffer_load_dword v44, off, s[0:3], s33 offset:40 ; 4-byte Folded Reload +; GENERIC-NEXT: buffer_load_dword v43, off, s[0:3], s33 offset:44 ; 4-byte Folded Reload +; GENERIC-NEXT: buffer_load_dword v42, off, s[0:3], s33 offset:48 ; 4-byte Folded Reload +; GENERIC-NEXT: buffer_load_dword v41, off, s[0:3], s33 offset:52 ; 4-byte Folded Reload +; GENERIC-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:56 ; 4-byte Folded Reload +; GENERIC-NEXT: v_readlane_b32 s30, v63, 0 +; GENERIC-NEXT: v_readlane_b32 s31, v63, 1 +; GENERIC-NEXT: s_mov_b32 s32, s34 +; GENERIC-NEXT: s_mov_b32 s34, s38 +; GENERIC-NEXT: s_or_saveexec_b64 s[36:37], -1 +; GENERIC-NEXT: buffer_load_dword v63, off, s[0:3], s33 offset:1564 ; 4-byte Folded Reload +; GENERIC-NEXT: s_mov_b64 exec, s[36:37] +; GENERIC-NEXT: s_mov_b32 s33, s35 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: s_setpc_b64 s[30:31] +; +; GENERIC-GCNTRACKERS-LABEL: call_72xi32: +; GENERIC-GCNTRACKERS: ; %bb.0: ; %entry +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s35, s33 +; GENERIC-GCNTRACKERS-NEXT: s_add_i32 s33, s32, 0x7fc0 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s33, s33, 0xffff8000 +; GENERIC-GCNTRACKERS-NEXT: s_or_saveexec_b64 s[36:37], -1 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v63, off, s[0:3], s33 offset:1564 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: s_mov_b64 exec, s[36:37] +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s38, s34 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s34, s32 +; GENERIC-GCNTRACKERS-NEXT: s_add_i32 s32, s32, 0x28000 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v40, off, s[0:3], s33 offset:56 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v41, off, s[0:3], s33 offset:52 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v42, off, s[0:3], s33 offset:48 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v43, off, s[0:3], s33 offset:44 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v44, off, s[0:3], s33 offset:40 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v45, off, s[0:3], s33 offset:36 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v46, off, s[0:3], s33 offset:32 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v47, off, s[0:3], s33 offset:28 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v56, off, s[0:3], s33 offset:24 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v57, off, s[0:3], s33 offset:20 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v58, off, s[0:3], s33 offset:16 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v59, off, s[0:3], s33 offset:12 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v60, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v61, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v62, off, s[0:3], s33 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: v_writelane_b32 v63, s30, 0 +; GENERIC-GCNTRACKERS-NEXT: v_writelane_b32 v63, s31, 1 +; GENERIC-GCNTRACKERS-NEXT: s_getpc_b64 s[36:37] +; GENERIC-GCNTRACKERS-NEXT: s_add_u32 s36, s36, return_72xi32@gotpcrel32@lo+4 +; GENERIC-GCNTRACKERS-NEXT: s_addc_u32 s37, s37, return_72xi32@gotpcrel32@hi+12 +; GENERIC-GCNTRACKERS-NEXT: s_load_dwordx2 s[36:37], s[36:37], 0x0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, 0 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:8 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:12 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:16 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:20 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:28 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:32 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:36 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:40 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:44 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:48 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:52 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_lshr_b32_e64 v0, s33, 6 +; GENERIC-GCNTRACKERS-NEXT: s_movk_i32 vcc_lo, 0x200 +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v0, vcc, vcc_lo, v0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v6, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v7, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v8, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v9, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v10, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v11, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v12, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v13, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v14, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v15, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v16, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v17, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v18, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v19, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v20, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v21, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v22, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v23, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v24, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v25, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v26, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v27, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v28, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v29, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v30, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v31, 0 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:636 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v1, off, s[0:3], s33 offset:640 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, off, s[0:3], s33 offset:644 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v33, off, s[0:3], s33 offset:648 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v34, off, s[0:3], s33 offset:652 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v35, off, s[0:3], s33 offset:656 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v36, off, s[0:3], s33 offset:660 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v37, off, s[0:3], s33 offset:664 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v38, off, s[0:3], s33 offset:668 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v39, off, s[0:3], s33 offset:672 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v48, off, s[0:3], s33 offset:676 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v49, off, s[0:3], s33 offset:680 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v50, off, s[0:3], s33 offset:684 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v51, off, s[0:3], s33 offset:688 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v52, off, s[0:3], s33 offset:692 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v53, off, s[0:3], s33 offset:696 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v54, off, s[0:3], s33 offset:700 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v55, off, s[0:3], s33 offset:704 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:708 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v41, off, s[0:3], s33 offset:712 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v42, off, s[0:3], s33 offset:716 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v43, off, s[0:3], s33 offset:720 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v44, off, s[0:3], s33 offset:724 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v45, off, s[0:3], s33 offset:728 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v46, off, s[0:3], s33 offset:732 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v47, off, s[0:3], s33 offset:736 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v56, off, s[0:3], s33 offset:740 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v57, off, s[0:3], s33 offset:748 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v58, off, s[0:3], s33 offset:752 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v59, off, s[0:3], s33 offset:756 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v60, off, s[0:3], s33 offset:760 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v61, off, s[0:3], s33 offset:764 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v62, off, s[0:3], s33 offset:768 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:772 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v3, off, s[0:3], s33 offset:776 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v4, off, s[0:3], s33 offset:780 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v5, off, s[0:3], s33 offset:784 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v6, off, s[0:3], s33 offset:788 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v7, off, s[0:3], s33 offset:792 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v8, off, s[0:3], s33 offset:796 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v12, off, s[0:3], s33 offset:516 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v13, off, s[0:3], s33 offset:520 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v14, off, s[0:3], s33 offset:524 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v15, off, s[0:3], s33 offset:528 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v16, off, s[0:3], s33 offset:532 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v17, off, s[0:3], s33 offset:536 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v18, off, s[0:3], s33 offset:540 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:544 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:548 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v11, off, s[0:3], s33 offset:552 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:556 +; GENERIC-GCNTRACKERS-NEXT: s_movk_i32 vcc_lo, 0x400 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v19, off, s[0:3], s33 offset:1540 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:560 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v19, off, s[0:3], s33 offset:1544 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:564 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v19, off, s[0:3], s33 offset:1548 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:568 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v19, off, s[0:3], s33 offset:1552 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:572 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v19, off, s[0:3], s33 offset:1556 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:576 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v19, off, s[0:3], s33 offset:1560 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:580 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v19, off, s[0:3], s33 offset:1536 ; 4-byte Folded Spill +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:584 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:588 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v21, off, s[0:3], s33 offset:592 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v22, off, s[0:3], s33 offset:596 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v23, off, s[0:3], s33 offset:600 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v24, off, s[0:3], s33 offset:604 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v25, off, s[0:3], s33 offset:608 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v26, off, s[0:3], s33 offset:612 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v27, off, s[0:3], s33 offset:616 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v28, off, s[0:3], s33 offset:620 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v29, off, s[0:3], s33 offset:624 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v30, off, s[0:3], s33 offset:628 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v31, off, s[0:3], s33 offset:632 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:8 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:12 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:16 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:20 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:24 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:28 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:32 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:36 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:40 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:44 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:48 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:52 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:56 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:60 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:64 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:68 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:72 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:76 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:80 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:84 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:88 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:92 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:96 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:100 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:104 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, 24 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:112 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:116 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:120 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:124 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:128 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:132 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:136 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:140 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:144 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:148 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:152 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:156 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:160 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(6) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, v12 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(5) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, v13 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(4) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, v14 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(3) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, v15 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(2) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v6, v16 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(1) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v7, v17 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v8, v18 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v12, off, s[0:3], s33 offset:1540 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v13, off, s[0:3], s33 offset:1544 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v14, off, s[0:3], s33 offset:1548 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v15, off, s[0:3], s33 offset:1552 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v16, off, s[0:3], s33 offset:1556 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v17, off, s[0:3], s33 offset:1560 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v18, off, s[0:3], s33 offset:1536 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: v_lshr_b32_e64 v0, s33, 6 +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v0, vcc, vcc_lo, v0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, 42 +; GENERIC-GCNTRACKERS-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v62, off, s[0:3], s33 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v61, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v60, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v59, off, s[0:3], s33 offset:12 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v58, off, s[0:3], s33 offset:16 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v57, off, s[0:3], s33 offset:20 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v56, off, s[0:3], s33 offset:24 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v47, off, s[0:3], s33 offset:28 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v46, off, s[0:3], s33 offset:32 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v45, off, s[0:3], s33 offset:36 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v44, off, s[0:3], s33 offset:40 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v43, off, s[0:3], s33 offset:44 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v42, off, s[0:3], s33 offset:48 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v41, off, s[0:3], s33 offset:52 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:56 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: v_readlane_b32 s30, v63, 0 +; GENERIC-GCNTRACKERS-NEXT: v_readlane_b32 s31, v63, 1 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s32, s34 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s34, s38 +; GENERIC-GCNTRACKERS-NEXT: s_or_saveexec_b64 s[36:37], -1 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v63, off, s[0:3], s33 offset:1564 ; 4-byte Folded Reload +; GENERIC-GCNTRACKERS-NEXT: s_mov_b64 exec, s[36:37] +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s33, s35 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: s_setpc_b64 s[30:31] entry: %ret.0 = call amdgpu_gfx <72 x i32> @return_72xi32(<72 x i32> zeroinitializer) %val.0 = insertelement <72 x i32> %ret.0, i32 42, i32 0 @@ -51,6 +3590,530 @@ define amdgpu_kernel void @global_extload_v16f16_to_v16f64(ptr addrspace(1) %out, ptr addrspace(1) %in) { +; GFX11-PAL-LABEL: global_extload_v16f16_to_v16f64: +; GFX11-PAL: ; %bb.0: +; GFX11-PAL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 +; GFX11-PAL-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PAL-NEXT: s_load_b256 s[4:11], s[2:3], 0x0 +; GFX11-PAL-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v14, s9 +; GFX11-PAL-NEXT: s_lshr_b32 s9, s9, 16 +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v11, s8 +; GFX11-PAL-NEXT: s_lshr_b32 s8, s8, 16 +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v2, s5 +; GFX11-PAL-NEXT: s_lshr_b32 s3, s5, 16 +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v6, s7 +; GFX11-PAL-NEXT: s_lshr_b32 s5, s7, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s7, s11, 16 +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v23, s9 +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v0, s4 +; GFX11-PAL-NEXT: s_lshr_b32 s2, s4, 16 +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v3, s6 +; GFX11-PAL-NEXT: s_lshr_b32 s4, s6, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s6, s10, 16 +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v22, s8 +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v10, s11 +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v19, s7 +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v7, s10 +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v18, s6 +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v15, s5 +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v34, s4 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[28:29], v14 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[30:31], v23 +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v33, s3 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[24:25], v11 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[26:27], v22 +; GFX11-PAL-NEXT: v_cvt_f32_f16_e32 v32, s2 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[20:21], v10 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[22:23], v19 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[16:17], v7 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[18:19], v18 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[12:13], v6 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[14:15], v15 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[8:9], v3 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[10:11], v34 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[4:5], v2 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[6:7], v33 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[0:1], v0 +; GFX11-PAL-NEXT: v_cvt_f64_f32_e32 v[2:3], v32 +; GFX11-PAL-NEXT: v_mov_b32_e32 v32, 0 +; GFX11-PAL-NEXT: s_clause 0x7 +; GFX11-PAL-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:80 +; GFX11-PAL-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:64 +; GFX11-PAL-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:112 +; GFX11-PAL-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:96 +; GFX11-PAL-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:48 +; GFX11-PAL-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:32 +; GFX11-PAL-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:16 +; GFX11-PAL-NEXT: global_store_b128 v32, v[0:3], s[0:1] +; GFX11-PAL-NEXT: s_endpgm +; +; GFX11-PAL-GCNTRACKERS-LABEL: global_extload_v16f16_to_v16f64: +; GFX11-PAL-GCNTRACKERS: ; %bb.0: +; GFX11-PAL-GCNTRACKERS-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: s_load_b256 s[4:11], s[2:3], 0x0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v14, s9 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s9, s9, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v11, s8 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s8, s8, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v2, s5 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s3, s5, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v6, s7 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s5, s7, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s7, s11, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v23, s9 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v0, s4 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s2, s4, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v3, s6 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s4, s6, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s6, s10, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v22, s8 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v10, s11 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v19, s7 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v7, s10 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v18, s6 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v15, s5 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v34, s4 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[28:29], v14 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[30:31], v23 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v33, s3 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[24:25], v11 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[26:27], v22 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v32, s2 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[20:21], v10 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[22:23], v19 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[16:17], v7 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[18:19], v18 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[12:13], v6 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[14:15], v15 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[8:9], v3 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[10:11], v34 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[4:5], v2 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[6:7], v33 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[0:1], v0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[2:3], v32 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v32, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x7 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:80 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:64 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:112 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:96 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:48 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:32 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:16 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v32, v[0:3], s[0:1] +; GFX11-PAL-GCNTRACKERS-NEXT: s_endpgm +; +; TONGA-LABEL: global_extload_v16f16_to_v16f64: +; TONGA: ; %bb.0: +; TONGA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; TONGA-NEXT: s_waitcnt lgkmcnt(0) +; TONGA-NEXT: v_mov_b32_e32 v0, s2 +; TONGA-NEXT: v_mov_b32_e32 v1, s3 +; TONGA-NEXT: flat_load_dwordx4 v[4:7], v[0:1] +; TONGA-NEXT: s_add_u32 s2, s2, 16 +; TONGA-NEXT: s_addc_u32 s3, s3, 0 +; TONGA-NEXT: v_mov_b32_e32 v0, s2 +; TONGA-NEXT: v_mov_b32_e32 v1, s3 +; TONGA-NEXT: flat_load_dwordx4 v[0:3], v[0:1] +; TONGA-NEXT: s_add_u32 s2, s0, 48 +; TONGA-NEXT: s_addc_u32 s3, s1, 0 +; TONGA-NEXT: v_mov_b32_e32 v14, s3 +; TONGA-NEXT: v_mov_b32_e32 v13, s2 +; TONGA-NEXT: s_add_u32 s2, s0, 32 +; TONGA-NEXT: s_addc_u32 s3, s1, 0 +; TONGA-NEXT: v_mov_b32_e32 v16, s3 +; TONGA-NEXT: v_mov_b32_e32 v15, s2 +; TONGA-NEXT: s_add_u32 s2, s0, 16 +; TONGA-NEXT: s_addc_u32 s3, s1, 0 +; TONGA-NEXT: v_mov_b32_e32 v18, s3 +; TONGA-NEXT: v_mov_b32_e32 v17, s2 +; TONGA-NEXT: v_mov_b32_e32 v12, s1 +; TONGA-NEXT: v_mov_b32_e32 v11, s0 +; TONGA-NEXT: s_add_u32 s2, s0, 0x50 +; TONGA-NEXT: s_addc_u32 s3, s1, 0 +; TONGA-NEXT: s_waitcnt vmcnt(1) +; TONGA-NEXT: v_cvt_f32_f16_e32 v8, v7 +; TONGA-NEXT: v_cvt_f32_f16_sdwa v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[7:8], v8 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[9:10], v9 +; TONGA-NEXT: flat_store_dwordx4 v[13:14], v[7:10] +; TONGA-NEXT: v_mov_b32_e32 v14, s3 +; TONGA-NEXT: v_cvt_f32_f16_e32 v7, v6 +; TONGA-NEXT: v_cvt_f32_f16_sdwa v8, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-NEXT: s_waitcnt vmcnt(1) +; TONGA-NEXT: v_cvt_f32_f16_e32 v10, v2 +; TONGA-NEXT: v_mov_b32_e32 v13, s2 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[6:7], v7 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[8:9], v8 +; TONGA-NEXT: s_add_u32 s2, s0, 64 +; TONGA-NEXT: s_addc_u32 s3, s1, 0 +; TONGA-NEXT: flat_store_dwordx4 v[15:16], v[6:9] +; TONGA-NEXT: v_mov_b32_e32 v16, s3 +; TONGA-NEXT: v_cvt_f32_f16_e32 v6, v5 +; TONGA-NEXT: v_cvt_f32_f16_sdwa v7, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-NEXT: v_cvt_f32_f16_e32 v8, v4 +; TONGA-NEXT: v_cvt_f32_f16_sdwa v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[4:5], v6 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[6:7], v7 +; TONGA-NEXT: v_mov_b32_e32 v15, s2 +; TONGA-NEXT: s_add_u32 s2, s0, 0x70 +; TONGA-NEXT: s_addc_u32 s3, s1, 0 +; TONGA-NEXT: flat_store_dwordx4 v[17:18], v[4:7] +; TONGA-NEXT: v_cvt_f32_f16_sdwa v17, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[4:5], v8 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[6:7], v9 +; TONGA-NEXT: v_cvt_f32_f16_sdwa v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-NEXT: v_cvt_f32_f16_sdwa v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-NEXT: v_cvt_f32_f16_e32 v2, v1 +; TONGA-NEXT: flat_store_dwordx4 v[11:12], v[4:7] +; TONGA-NEXT: v_cvt_f32_f16_sdwa v11, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-NEXT: v_cvt_f32_f16_e32 v7, v3 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[3:4], v9 +; TONGA-NEXT: v_cvt_f32_f16_e32 v9, v0 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[1:2], v2 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[5:6], v10 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[11:12], v11 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[9:10], v9 +; TONGA-NEXT: flat_store_dwordx4 v[13:14], v[1:4] +; TONGA-NEXT: s_add_u32 s0, s0, 0x60 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[0:1], v7 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[2:3], v17 +; TONGA-NEXT: v_cvt_f64_f32_e32 v[7:8], v8 +; TONGA-NEXT: s_addc_u32 s1, s1, 0 +; TONGA-NEXT: v_mov_b32_e32 v20, s3 +; TONGA-NEXT: v_mov_b32_e32 v19, s2 +; TONGA-NEXT: v_mov_b32_e32 v14, s1 +; TONGA-NEXT: v_mov_b32_e32 v13, s0 +; TONGA-NEXT: flat_store_dwordx4 v[15:16], v[9:12] +; TONGA-NEXT: flat_store_dwordx4 v[19:20], v[0:3] +; TONGA-NEXT: flat_store_dwordx4 v[13:14], v[5:8] +; TONGA-NEXT: s_endpgm +; +; TONGA-GCNTRACKERS-LABEL: global_extload_v16f16_to_v16f64: +; TONGA-GCNTRACKERS: ; %bb.0: +; TONGA-GCNTRACKERS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s2 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s3 +; TONGA-GCNTRACKERS-NEXT: flat_load_dwordx4 v[4:7], v[0:1] +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s2, s2, 16 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s3, s3, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s2 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s3 +; TONGA-GCNTRACKERS-NEXT: flat_load_dwordx4 v[0:3], v[0:1] +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s2, s0, 48 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s3, s1, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v14, s3 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v13, s2 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s2, s0, 32 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s3, s1, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v16, s3 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v15, s2 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s2, s0, 16 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s3, s1, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v18, s3 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v17, s2 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s2, s0, 0x50 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v12, s1 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v11, s0 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s3, s1, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v20, s3 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v19, s2 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s2, s0, 64 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s3, s1, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v22, s3 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v21, s2 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s2, s0, 0x70 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s3, s1, 0 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s0, s0, 0x60 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s1, s1, 0 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(1) +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v8, v7 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_sdwa v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[7:8], v8 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[9:10], v9 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[13:14], v[7:10] +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(1) +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v13, v1 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v7, v6 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_sdwa v8, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v10, v0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v14, s2 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[6:7], v7 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[8:9], v8 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[15:16], v[6:9] +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_sdwa v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v6, v5 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_sdwa v7, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v9, v4 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_sdwa v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[5:6], v6 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[7:8], v7 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v15, s3 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[17:18], v[5:8] +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_sdwa v17, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v6, v2 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v7, v3 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_sdwa v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[2:3], v9 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[4:5], v4 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f32_f16_sdwa v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[0:1], v6 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[6:7], v7 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[11:12], v[2:5] +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[10:11], v10 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[2:3], v13 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[4:5], v17 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[12:13], v9 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[8:9], v8 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[19:20], v[2:5] +; TONGA-GCNTRACKERS-NEXT: s_nop 0 +; TONGA-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[2:3], v16 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[21:22], v[10:13] +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[14:15], v[6:9] +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: s_endpgm +; +; GFX908-LABEL: global_extload_v16f16_to_v16f64: +; GFX908: ; %bb.0: +; GFX908-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24 +; GFX908-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: v_cvt_f32_f16_e32 v6, s5 +; GFX908-NEXT: s_lshr_b32 s5, s5, 16 +; GFX908-NEXT: v_cvt_f32_f16_e32 v2, s1 +; GFX908-NEXT: v_cvt_f32_f16_e32 v8, s5 +; GFX908-NEXT: s_lshr_b32 s1, s1, 16 +; GFX908-NEXT: v_cvt_f32_f16_e32 v18, s1 +; GFX908-NEXT: s_lshr_b32 s1, s4, 16 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[4:5], v2 +; GFX908-NEXT: v_cvt_f32_f16_e32 v2, s4 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[6:7], v6 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[8:9], v8 +; GFX908-NEXT: v_cvt_f32_f16_e32 v10, s1 +; GFX908-NEXT: s_lshr_b32 s1, s7, 16 +; GFX908-NEXT: v_cvt_f32_f16_e32 v11, s7 +; GFX908-NEXT: global_store_dwordx4 v16, v[6:9], s[8:9] offset:80 +; GFX908-NEXT: v_cvt_f32_f16_e32 v0, s0 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[6:7], v2 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[8:9], v10 +; GFX908-NEXT: v_cvt_f32_f16_e32 v2, s1 +; GFX908-NEXT: s_lshr_b32 s1, s6, 16 +; GFX908-NEXT: v_cvt_f32_f16_e32 v10, s6 +; GFX908-NEXT: global_store_dwordx4 v16, v[6:9], s[8:9] offset:64 +; GFX908-NEXT: s_lshr_b32 s0, s0, 16 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[6:7], v11 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[8:9], v2 +; GFX908-NEXT: v_cvt_f32_f16_e32 v2, s1 +; GFX908-NEXT: s_lshr_b32 s1, s3, 16 +; GFX908-NEXT: v_cvt_f32_f16_e32 v17, s0 +; GFX908-NEXT: s_lshr_b32 s0, s2, 16 +; GFX908-NEXT: v_cvt_f32_f16_e32 v11, s3 +; GFX908-NEXT: global_store_dwordx4 v16, v[6:9], s[8:9] offset:112 +; GFX908-NEXT: v_cvt_f32_f16_e32 v3, s2 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[6:7], v10 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[8:9], v2 +; GFX908-NEXT: v_cvt_f32_f16_e32 v2, s1 +; GFX908-NEXT: v_cvt_f32_f16_e32 v14, s0 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[12:13], v3 +; GFX908-NEXT: global_store_dwordx4 v16, v[6:9], s[8:9] offset:96 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[0:1], v0 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[8:9], v11 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[10:11], v2 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[14:15], v14 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[6:7], v18 +; GFX908-NEXT: v_cvt_f64_f32_e32 v[2:3], v17 +; GFX908-NEXT: global_store_dwordx4 v16, v[8:11], s[8:9] offset:48 +; GFX908-NEXT: global_store_dwordx4 v16, v[12:15], s[8:9] offset:32 +; GFX908-NEXT: global_store_dwordx4 v16, v[4:7], s[8:9] offset:16 +; GFX908-NEXT: global_store_dwordx4 v16, v[0:3], s[8:9] +; GFX908-NEXT: s_endpgm +; +; GFX908-GCNTRACKERS-LABEL: global_extload_v16f16_to_v16f64: +; GFX908-GCNTRACKERS: ; %bb.0: +; GFX908-GCNTRACKERS-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v20, 0 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GCNTRACKERS-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v2, s1 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s1, s1, 16 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v6, s1 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s1, s5, 16 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v10, s5 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v12, s1 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s1, s4, 16 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v15, s4 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[10:11], v10 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[12:13], v12 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v16, s1 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s1, s7, 16 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v3, s2 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v14, s7 +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v20, v[10:13], s[8:9] offset:80 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v0, s0 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[10:11], v15 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[12:13], v16 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v16, s1 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s0, s0, 16 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v21, s0 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s0, s3, 16 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s1, s6, 16 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s2, s2, 16 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[4:5], v2 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[8:9], v3 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v2, s3 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v3, s6 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[14:15], v14 +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v20, v[10:13], s[8:9] offset:64 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[16:17], v16 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v10, s1 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v11, s0 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v22, s2 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[12:13], v3 +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v20, v[14:17], s[8:9] offset:112 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[18:19], v11 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[14:15], v10 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[16:17], v2 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[10:11], v22 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[6:7], v6 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[0:1], v0 +; GFX908-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[2:3], v21 +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v20, v[12:15], s[8:9] offset:96 +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v20, v[16:19], s[8:9] offset:48 +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v20, v[8:11], s[8:9] offset:32 +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v20, v[4:7], s[8:9] offset:16 +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v20, v[0:3], s[8:9] +; GFX908-GCNTRACKERS-NEXT: s_endpgm +; +; GENERIC-LABEL: global_extload_v16f16_to_v16f64: +; GENERIC: ; %bb.0: +; GENERIC-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9 +; GENERIC-NEXT: s_waitcnt lgkmcnt(0) +; GENERIC-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0 +; GENERIC-NEXT: s_mov_b32 s11, 0xf000 +; GENERIC-NEXT: s_mov_b32 s10, -1 +; GENERIC-NEXT: s_waitcnt lgkmcnt(0) +; GENERIC-NEXT: s_lshr_b32 s12, s7, 16 +; GENERIC-NEXT: v_cvt_f32_f16_e32 v8, s7 +; GENERIC-NEXT: v_cvt_f32_f16_e32 v10, s12 +; GENERIC-NEXT: s_lshr_b32 s13, s6, 16 +; GENERIC-NEXT: v_cvt_f32_f16_e32 v13, s6 +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[8:9], v8 +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[10:11], v10 +; GENERIC-NEXT: v_cvt_f32_f16_e32 v14, s13 +; GENERIC-NEXT: s_lshr_b32 s14, s5, 16 +; GENERIC-NEXT: v_cvt_f32_f16_e32 v12, s5 +; GENERIC-NEXT: v_cvt_f32_f16_e32 v15, s14 +; GENERIC-NEXT: buffer_store_dwordx4 v[8:11], off, s[8:11], 0 offset:112 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[8:9], v13 +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[10:11], v14 +; GENERIC-NEXT: s_lshr_b32 s15, s4, 16 +; GENERIC-NEXT: s_lshr_b32 s16, s3, 16 +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[12:13], v12 +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[14:15], v15 +; GENERIC-NEXT: buffer_store_dwordx4 v[8:11], off, s[8:11], 0 offset:96 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_cvt_f32_f16_e32 v10, s4 +; GENERIC-NEXT: v_cvt_f32_f16_e32 v11, s15 +; GENERIC-NEXT: s_lshr_b32 s17, s0, 16 +; GENERIC-NEXT: v_cvt_f32_f16_e32 v0, s0 +; GENERIC-NEXT: s_lshr_b32 s0, s2, 16 +; GENERIC-NEXT: v_cvt_f32_f16_e32 v17, s3 +; GENERIC-NEXT: v_cvt_f32_f16_e32 v19, s16 +; GENERIC-NEXT: v_cvt_f32_f16_e32 v2, s17 +; GENERIC-NEXT: s_lshr_b32 s17, s1, 16 +; GENERIC-NEXT: v_cvt_f32_f16_e32 v4, s2 +; GENERIC-NEXT: v_cvt_f32_f16_e32 v6, s0 +; GENERIC-NEXT: v_cvt_f32_f16_e32 v16, s1 +; GENERIC-NEXT: v_cvt_f32_f16_e32 v18, s17 +; GENERIC-NEXT: buffer_store_dwordx4 v[12:15], off, s[8:11], 0 offset:80 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[12:13], v10 +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[14:15], v11 +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[8:9], v17 +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[10:11], v19 +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[4:5], v4 +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[6:7], v6 +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[16:17], v16 +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[18:19], v18 +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[0:1], v0 +; GENERIC-NEXT: v_cvt_f64_f32_e32 v[2:3], v2 +; GENERIC-NEXT: buffer_store_dwordx4 v[12:15], off, s[8:11], 0 offset:64 +; GENERIC-NEXT: buffer_store_dwordx4 v[8:11], off, s[8:11], 0 offset:48 +; GENERIC-NEXT: buffer_store_dwordx4 v[4:7], off, s[8:11], 0 offset:32 +; GENERIC-NEXT: buffer_store_dwordx4 v[16:19], off, s[8:11], 0 offset:16 +; GENERIC-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 +; GENERIC-NEXT: s_endpgm +; +; GENERIC-GCNTRACKERS-LABEL: global_extload_v16f16_to_v16f64: +; GENERIC-GCNTRACKERS: ; %bb.0: +; GENERIC-GCNTRACKERS-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s11, 0xf000 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s10, -1 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v6, s2 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s12, s7, 16 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v11, s7 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v16, s12 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[8:9], v6 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v6, s3 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s13, s6, 16 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s14, s5, 16 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[14:15], v11 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[12:13], v6 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v6, s6 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[16:17], v16 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v11, s13 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v22, s5 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[18:19], v6 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v6, s14 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s15, s4, 16 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s16, s3, 16 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[14:17], off, s[8:11], 0 offset:112 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[20:21], v11 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v11, s4 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[16:17], v6 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v6, s15 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s17, s2, 16 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[14:15], v22 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v22, s16 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s18, s1, 16 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v10, s17 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v0, s0 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s0, s0, 16 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v4, s1 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v7, s18 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f32_f16_e32 v2, s0 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[18:21], off, s[8:11], 0 offset:96 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[18:19], v11 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[20:21], v6 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[14:17], off, s[8:11], 0 offset:80 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[14:15], v22 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[10:11], v10 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[4:5], v4 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[6:7], v7 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[0:1], v0 +; GENERIC-GCNTRACKERS-NEXT: v_cvt_f64_f32_e32 v[2:3], v2 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[18:21], off, s[8:11], 0 offset:64 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[12:15], off, s[8:11], 0 offset:48 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[8:11], off, s[8:11], 0 offset:32 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[4:7], off, s[8:11], 0 offset:16 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 +; GENERIC-GCNTRACKERS-NEXT: s_endpgm %val = load <16 x half>, ptr addrspace(1) %in %cvt = fpext <16 x half> %val to <16 x double> store <16 x double> %cvt, ptr addrspace(1) %out @@ -66,6 +4129,1343 @@ ; GENERIC-GCNTRACKERS: Occupancy: 10 define amdgpu_kernel void @constant_zextload_v64i16_to_v64i32(ptr addrspace(1) %out, ptr addrspace(4) %in) { +; GFX11-PAL-LABEL: constant_zextload_v64i16_to_v64i32: +; GFX11-PAL: ; %bb.0: +; GFX11-PAL-NEXT: s_load_b128 s[36:39], s[4:5], 0x0 +; GFX11-PAL-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PAL-NEXT: s_clause 0x1 +; GFX11-PAL-NEXT: s_load_b512 s[16:31], s[38:39], 0x0 +; GFX11-PAL-NEXT: s_load_b512 s[0:15], s[38:39], 0x40 +; GFX11-PAL-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PAL-NEXT: s_lshr_b32 s49, s31, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s65, s15, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s66, s14, 16 +; GFX11-PAL-NEXT: s_and_b32 s14, s14, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s15, s15, 0xffff +; GFX11-PAL-NEXT: s_lshr_b32 s63, s13, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s64, s12, 16 +; GFX11-PAL-NEXT: s_and_b32 s13, s13, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s12, s12, 0xffff +; GFX11-PAL-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v1, s66 +; GFX11-PAL-NEXT: s_lshr_b32 s61, s11, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s62, s10, 16 +; GFX11-PAL-NEXT: s_and_b32 s11, s11, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s10, s10, 0xffff +; GFX11-PAL-NEXT: v_dual_mov_b32 v0, s14 :: v_dual_mov_b32 v3, s65 +; GFX11-PAL-NEXT: v_dual_mov_b32 v2, s15 :: v_dual_mov_b32 v5, s64 +; GFX11-PAL-NEXT: s_lshr_b32 s59, s9, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s60, s8, 16 +; GFX11-PAL-NEXT: s_and_b32 s9, s9, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s8, s8, 0xffff +; GFX11-PAL-NEXT: v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v7, s63 +; GFX11-PAL-NEXT: v_dual_mov_b32 v6, s13 :: v_dual_mov_b32 v9, s62 +; GFX11-PAL-NEXT: v_dual_mov_b32 v8, s10 :: v_dual_mov_b32 v11, s61 +; GFX11-PAL-NEXT: v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v13, s60 +; GFX11-PAL-NEXT: s_lshr_b32 s57, s7, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s58, s6, 16 +; GFX11-PAL-NEXT: s_and_b32 s7, s7, 0xffff +; GFX11-PAL-NEXT: v_dual_mov_b32 v12, s8 :: v_dual_mov_b32 v15, s59 +; GFX11-PAL-NEXT: v_mov_b32_e32 v14, s9 +; GFX11-PAL-NEXT: s_and_b32 s6, s6, 0xffff +; GFX11-PAL-NEXT: s_lshr_b32 s55, s5, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s56, s4, 16 +; GFX11-PAL-NEXT: s_and_b32 s5, s5, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s4, s4, 0xffff +; GFX11-PAL-NEXT: s_lshr_b32 s53, s3, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s54, s2, 16 +; GFX11-PAL-NEXT: s_and_b32 s3, s3, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s2, s2, 0xffff +; GFX11-PAL-NEXT: s_clause 0x3 +; GFX11-PAL-NEXT: global_store_b128 v24, v[0:3], s[36:37] offset:240 +; GFX11-PAL-NEXT: global_store_b128 v24, v[4:7], s[36:37] offset:224 +; GFX11-PAL-NEXT: global_store_b128 v24, v[8:11], s[36:37] offset:208 +; GFX11-PAL-NEXT: global_store_b128 v24, v[12:15], s[36:37] offset:192 +; GFX11-PAL-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v3, s57 +; GFX11-PAL-NEXT: v_dual_mov_b32 v1, s58 :: v_dual_mov_b32 v2, s7 +; GFX11-PAL-NEXT: v_mov_b32_e32 v5, s56 +; GFX11-PAL-NEXT: s_lshr_b32 s51, s1, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s52, s0, 16 +; GFX11-PAL-NEXT: s_and_b32 s1, s1, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s0, s0, 0xffff +; GFX11-PAL-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v7, s55 +; GFX11-PAL-NEXT: v_dual_mov_b32 v6, s5 :: v_dual_mov_b32 v9, s54 +; GFX11-PAL-NEXT: s_lshr_b32 s50, s30, 16 +; GFX11-PAL-NEXT: s_and_b32 s31, s31, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s30, s30, 0xffff +; GFX11-PAL-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v11, s53 +; GFX11-PAL-NEXT: v_dual_mov_b32 v10, s3 :: v_dual_mov_b32 v13, s52 +; GFX11-PAL-NEXT: s_lshr_b32 s45, s27, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s46, s26, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s47, s29, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s48, s28, 16 +; GFX11-PAL-NEXT: s_and_b32 s27, s27, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s26, s26, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s29, s29, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s28, s28, 0xffff +; GFX11-PAL-NEXT: v_dual_mov_b32 v12, s0 :: v_dual_mov_b32 v15, s51 +; GFX11-PAL-NEXT: v_dual_mov_b32 v14, s1 :: v_dual_mov_b32 v17, s50 +; GFX11-PAL-NEXT: s_lshr_b32 s43, s25, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s44, s24, 16 +; GFX11-PAL-NEXT: s_and_b32 s25, s25, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s24, s24, 0xffff +; GFX11-PAL-NEXT: v_dual_mov_b32 v16, s30 :: v_dual_mov_b32 v19, s49 +; GFX11-PAL-NEXT: v_dual_mov_b32 v18, s31 :: v_dual_mov_b32 v21, s48 +; GFX11-PAL-NEXT: s_lshr_b32 s41, s23, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s42, s22, 16 +; GFX11-PAL-NEXT: s_and_b32 s23, s23, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s22, s22, 0xffff +; GFX11-PAL-NEXT: v_dual_mov_b32 v20, s28 :: v_dual_mov_b32 v23, s47 +; GFX11-PAL-NEXT: v_mov_b32_e32 v22, s29 +; GFX11-PAL-NEXT: s_clause 0x5 +; GFX11-PAL-NEXT: global_store_b128 v24, v[0:3], s[36:37] offset:176 +; GFX11-PAL-NEXT: global_store_b128 v24, v[4:7], s[36:37] offset:160 +; GFX11-PAL-NEXT: global_store_b128 v24, v[8:11], s[36:37] offset:144 +; GFX11-PAL-NEXT: global_store_b128 v24, v[12:15], s[36:37] offset:128 +; GFX11-PAL-NEXT: global_store_b128 v24, v[16:19], s[36:37] offset:112 +; GFX11-PAL-NEXT: global_store_b128 v24, v[20:23], s[36:37] offset:96 +; GFX11-PAL-NEXT: v_dual_mov_b32 v0, s26 :: v_dual_mov_b32 v3, s45 +; GFX11-PAL-NEXT: v_dual_mov_b32 v1, s46 :: v_dual_mov_b32 v2, s27 +; GFX11-PAL-NEXT: v_mov_b32_e32 v5, s44 +; GFX11-PAL-NEXT: s_lshr_b32 s39, s21, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s40, s20, 16 +; GFX11-PAL-NEXT: s_and_b32 s21, s21, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s20, s20, 0xffff +; GFX11-PAL-NEXT: v_dual_mov_b32 v4, s24 :: v_dual_mov_b32 v7, s43 +; GFX11-PAL-NEXT: v_dual_mov_b32 v6, s25 :: v_dual_mov_b32 v9, s42 +; GFX11-PAL-NEXT: s_lshr_b32 s35, s19, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s38, s18, 16 +; GFX11-PAL-NEXT: s_and_b32 s19, s19, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s18, s18, 0xffff +; GFX11-PAL-NEXT: v_dual_mov_b32 v8, s22 :: v_dual_mov_b32 v11, s41 +; GFX11-PAL-NEXT: v_dual_mov_b32 v10, s23 :: v_dual_mov_b32 v13, s40 +; GFX11-PAL-NEXT: s_lshr_b32 s33, s17, 16 +; GFX11-PAL-NEXT: s_lshr_b32 s34, s16, 16 +; GFX11-PAL-NEXT: s_and_b32 s17, s17, 0xffff +; GFX11-PAL-NEXT: s_and_b32 s16, s16, 0xffff +; GFX11-PAL-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v15, s39 +; GFX11-PAL-NEXT: v_dual_mov_b32 v14, s21 :: v_dual_mov_b32 v17, s38 +; GFX11-PAL-NEXT: v_dual_mov_b32 v16, s18 :: v_dual_mov_b32 v19, s35 +; GFX11-PAL-NEXT: v_dual_mov_b32 v18, s19 :: v_dual_mov_b32 v21, s34 +; GFX11-PAL-NEXT: v_dual_mov_b32 v20, s16 :: v_dual_mov_b32 v23, s33 +; GFX11-PAL-NEXT: v_mov_b32_e32 v22, s17 +; GFX11-PAL-NEXT: s_clause 0x5 +; GFX11-PAL-NEXT: global_store_b128 v24, v[0:3], s[36:37] offset:80 +; GFX11-PAL-NEXT: global_store_b128 v24, v[4:7], s[36:37] offset:64 +; GFX11-PAL-NEXT: global_store_b128 v24, v[8:11], s[36:37] offset:48 +; GFX11-PAL-NEXT: global_store_b128 v24, v[12:15], s[36:37] offset:32 +; GFX11-PAL-NEXT: global_store_b128 v24, v[16:19], s[36:37] offset:16 +; GFX11-PAL-NEXT: global_store_b128 v24, v[20:23], s[36:37] +; GFX11-PAL-NEXT: s_endpgm +; +; GFX11-PAL-GCNTRACKERS-LABEL: constant_zextload_v64i16_to_v64i32: +; GFX11-PAL-GCNTRACKERS: ; %bb.0: +; GFX11-PAL-GCNTRACKERS-NEXT: s_load_b128 s[36:39], s[4:5], 0x0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x1 +; GFX11-PAL-GCNTRACKERS-NEXT: s_load_b512 s[16:31], s[38:39], 0x0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_load_b512 s[0:15], s[38:39], 0x40 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s49, s31, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s65, s15, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s66, s14, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s14, s14, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s15, s15, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s63, s13, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s64, s12, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s13, s13, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s12, s12, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v1, s66 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s61, s11, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s62, s10, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s11, s11, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s10, s10, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v0, s14 :: v_dual_mov_b32 v3, s65 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v2, s15 :: v_dual_mov_b32 v5, s64 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s59, s9, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s60, s8, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s9, s9, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s8, s8, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v7, s63 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v6, s13 :: v_dual_mov_b32 v9, s62 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v8, s10 :: v_dual_mov_b32 v11, s61 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v13, s60 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s57, s7, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s58, s6, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s7, s7, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v12, s8 :: v_dual_mov_b32 v15, s59 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v14, s9 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s6, s6, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s55, s5, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s56, s4, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s5, s5, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s4, s4, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s53, s3, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s54, s2, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s3, s3, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s2, s2, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x3 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[0:3], s[36:37] offset:240 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[4:7], s[36:37] offset:224 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[8:11], s[36:37] offset:208 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[12:15], s[36:37] offset:192 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v3, s57 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v1, s58 :: v_dual_mov_b32 v2, s7 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s56 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s51, s1, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s52, s0, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s1, s1, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s0, s0, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v7, s55 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v6, s5 :: v_dual_mov_b32 v9, s54 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s50, s30, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s31, s31, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s30, s30, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v11, s53 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v10, s3 :: v_dual_mov_b32 v13, s52 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s45, s27, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s46, s26, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s47, s29, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s48, s28, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s27, s27, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s26, s26, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s29, s29, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s28, s28, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v12, s0 :: v_dual_mov_b32 v15, s51 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v14, s1 :: v_dual_mov_b32 v17, s50 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s43, s25, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s44, s24, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s25, s25, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s24, s24, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v16, s30 :: v_dual_mov_b32 v19, s49 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v18, s31 :: v_dual_mov_b32 v21, s48 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s41, s23, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s42, s22, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s23, s23, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s22, s22, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v20, s28 :: v_dual_mov_b32 v23, s47 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v22, s29 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x5 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[0:3], s[36:37] offset:176 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[4:7], s[36:37] offset:160 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[8:11], s[36:37] offset:144 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[12:15], s[36:37] offset:128 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[16:19], s[36:37] offset:112 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[20:23], s[36:37] offset:96 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v0, s26 :: v_dual_mov_b32 v3, s45 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v1, s46 :: v_dual_mov_b32 v2, s27 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s44 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s39, s21, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s40, s20, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s21, s21, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s20, s20, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v4, s24 :: v_dual_mov_b32 v7, s43 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v6, s25 :: v_dual_mov_b32 v9, s42 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s35, s19, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s38, s18, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s19, s19, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s18, s18, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v8, s22 :: v_dual_mov_b32 v11, s41 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v10, s23 :: v_dual_mov_b32 v13, s40 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s33, s17, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshr_b32 s34, s16, 16 +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s17, s17, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: s_and_b32 s16, s16, 0xffff +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v15, s39 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v14, s21 :: v_dual_mov_b32 v17, s38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v16, s18 :: v_dual_mov_b32 v19, s35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v18, s19 :: v_dual_mov_b32 v21, s34 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v20, s16 :: v_dual_mov_b32 v23, s33 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v22, s17 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x5 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[0:3], s[36:37] offset:80 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[4:7], s[36:37] offset:64 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[8:11], s[36:37] offset:48 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[12:15], s[36:37] offset:32 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[16:19], s[36:37] offset:16 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b128 v24, v[20:23], s[36:37] +; GFX11-PAL-GCNTRACKERS-NEXT: s_endpgm +; +; TONGA-LABEL: constant_zextload_v64i16_to_v64i32: +; TONGA: ; %bb.0: +; TONGA-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24 +; TONGA-NEXT: s_waitcnt lgkmcnt(0) +; TONGA-NEXT: s_load_dwordx16 s[16:31], s[38:39], 0x0 +; TONGA-NEXT: s_load_dwordx16 s[0:15], s[38:39], 0x40 +; TONGA-NEXT: s_waitcnt lgkmcnt(0) +; TONGA-NEXT: s_lshr_b32 s33, s17, 16 +; TONGA-NEXT: s_lshr_b32 s34, s16, 16 +; TONGA-NEXT: s_lshr_b32 s35, s19, 16 +; TONGA-NEXT: s_lshr_b32 s40, s18, 16 +; TONGA-NEXT: s_lshr_b32 s41, s21, 16 +; TONGA-NEXT: s_lshr_b32 s42, s20, 16 +; TONGA-NEXT: s_lshr_b32 s43, s23, 16 +; TONGA-NEXT: s_lshr_b32 s44, s22, 16 +; TONGA-NEXT: s_lshr_b32 s45, s25, 16 +; TONGA-NEXT: s_lshr_b32 s46, s24, 16 +; TONGA-NEXT: s_lshr_b32 s47, s27, 16 +; TONGA-NEXT: s_lshr_b32 s48, s26, 16 +; TONGA-NEXT: s_lshr_b32 s38, s29, 16 +; TONGA-NEXT: s_lshr_b32 s39, s28, 16 +; TONGA-NEXT: s_lshr_b32 s49, s31, 16 +; TONGA-NEXT: s_lshr_b32 s50, s30, 16 +; TONGA-NEXT: s_lshr_b32 s51, s1, 16 +; TONGA-NEXT: s_lshr_b32 s52, s0, 16 +; TONGA-NEXT: s_lshr_b32 s53, s3, 16 +; TONGA-NEXT: s_lshr_b32 s54, s2, 16 +; TONGA-NEXT: s_lshr_b32 s55, s5, 16 +; TONGA-NEXT: s_lshr_b32 s56, s4, 16 +; TONGA-NEXT: s_lshr_b32 s57, s7, 16 +; TONGA-NEXT: s_lshr_b32 s58, s6, 16 +; TONGA-NEXT: s_lshr_b32 s59, s9, 16 +; TONGA-NEXT: s_and_b32 s17, s17, 0xffff +; TONGA-NEXT: s_and_b32 s16, s16, 0xffff +; TONGA-NEXT: s_and_b32 s19, s19, 0xffff +; TONGA-NEXT: s_and_b32 s18, s18, 0xffff +; TONGA-NEXT: s_and_b32 s21, s21, 0xffff +; TONGA-NEXT: s_and_b32 s20, s20, 0xffff +; TONGA-NEXT: s_and_b32 s23, s23, 0xffff +; TONGA-NEXT: s_and_b32 s22, s22, 0xffff +; TONGA-NEXT: s_and_b32 s25, s25, 0xffff +; TONGA-NEXT: s_and_b32 s24, s24, 0xffff +; TONGA-NEXT: s_and_b32 s27, s27, 0xffff +; TONGA-NEXT: s_and_b32 s26, s26, 0xffff +; TONGA-NEXT: s_and_b32 s29, s29, 0xffff +; TONGA-NEXT: s_and_b32 s28, s28, 0xffff +; TONGA-NEXT: s_and_b32 s31, s31, 0xffff +; TONGA-NEXT: s_and_b32 s30, s30, 0xffff +; TONGA-NEXT: s_lshr_b32 s60, s8, 16 +; TONGA-NEXT: s_lshr_b32 s61, s11, 16 +; TONGA-NEXT: s_lshr_b32 s62, s10, 16 +; TONGA-NEXT: s_lshr_b32 s63, s13, 16 +; TONGA-NEXT: s_lshr_b32 s64, s12, 16 +; TONGA-NEXT: s_lshr_b32 s65, s15, 16 +; TONGA-NEXT: s_lshr_b32 s66, s14, 16 +; TONGA-NEXT: s_and_b32 s67, s1, 0xffff +; TONGA-NEXT: s_and_b32 s68, s0, 0xffff +; TONGA-NEXT: s_and_b32 s3, s3, 0xffff +; TONGA-NEXT: s_and_b32 s2, s2, 0xffff +; TONGA-NEXT: s_and_b32 s5, s5, 0xffff +; TONGA-NEXT: s_and_b32 s4, s4, 0xffff +; TONGA-NEXT: s_and_b32 s7, s7, 0xffff +; TONGA-NEXT: s_and_b32 s6, s6, 0xffff +; TONGA-NEXT: s_and_b32 s9, s9, 0xffff +; TONGA-NEXT: s_and_b32 s8, s8, 0xffff +; TONGA-NEXT: s_and_b32 s11, s11, 0xffff +; TONGA-NEXT: s_and_b32 s10, s10, 0xffff +; TONGA-NEXT: s_and_b32 s13, s13, 0xffff +; TONGA-NEXT: s_and_b32 s12, s12, 0xffff +; TONGA-NEXT: s_and_b32 s0, s15, 0xffff +; TONGA-NEXT: s_and_b32 s1, s14, 0xffff +; TONGA-NEXT: v_mov_b32_e32 v2, s0 +; TONGA-NEXT: s_add_u32 s0, s36, 0xf0 +; TONGA-NEXT: v_mov_b32_e32 v0, s1 +; TONGA-NEXT: s_addc_u32 s1, s37, 0 +; TONGA-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: s_add_u32 s0, s36, 0xe0 +; TONGA-NEXT: v_mov_b32_e32 v1, s66 +; TONGA-NEXT: v_mov_b32_e32 v3, s65 +; TONGA-NEXT: s_addc_u32 s1, s37, 0 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: s_add_u32 s0, s36, 0xd0 +; TONGA-NEXT: v_mov_b32_e32 v0, s12 +; TONGA-NEXT: v_mov_b32_e32 v1, s64 +; TONGA-NEXT: v_mov_b32_e32 v2, s13 +; TONGA-NEXT: v_mov_b32_e32 v3, s63 +; TONGA-NEXT: s_addc_u32 s1, s37, 0 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: s_add_u32 s0, s36, 0xc0 +; TONGA-NEXT: v_mov_b32_e32 v0, s10 +; TONGA-NEXT: v_mov_b32_e32 v1, s62 +; TONGA-NEXT: v_mov_b32_e32 v2, s11 +; TONGA-NEXT: v_mov_b32_e32 v3, s61 +; TONGA-NEXT: s_addc_u32 s1, s37, 0 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: s_add_u32 s0, s36, 0xb0 +; TONGA-NEXT: v_mov_b32_e32 v0, s8 +; TONGA-NEXT: v_mov_b32_e32 v1, s60 +; TONGA-NEXT: v_mov_b32_e32 v2, s9 +; TONGA-NEXT: v_mov_b32_e32 v3, s59 +; TONGA-NEXT: s_addc_u32 s1, s37, 0 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: s_add_u32 s0, s36, 0xa0 +; TONGA-NEXT: v_mov_b32_e32 v0, s6 +; TONGA-NEXT: v_mov_b32_e32 v1, s58 +; TONGA-NEXT: v_mov_b32_e32 v2, s7 +; TONGA-NEXT: v_mov_b32_e32 v3, s57 +; TONGA-NEXT: s_addc_u32 s1, s37, 0 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: s_add_u32 s0, s36, 0x90 +; TONGA-NEXT: v_mov_b32_e32 v0, s4 +; TONGA-NEXT: v_mov_b32_e32 v1, s56 +; TONGA-NEXT: v_mov_b32_e32 v2, s5 +; TONGA-NEXT: v_mov_b32_e32 v3, s55 +; TONGA-NEXT: s_addc_u32 s1, s37, 0 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: s_add_u32 s0, s36, 0x80 +; TONGA-NEXT: v_mov_b32_e32 v0, s2 +; TONGA-NEXT: v_mov_b32_e32 v1, s54 +; TONGA-NEXT: v_mov_b32_e32 v2, s3 +; TONGA-NEXT: v_mov_b32_e32 v3, s53 +; TONGA-NEXT: s_addc_u32 s1, s37, 0 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: s_add_u32 s0, s36, 0x70 +; TONGA-NEXT: v_mov_b32_e32 v0, s68 +; TONGA-NEXT: v_mov_b32_e32 v1, s52 +; TONGA-NEXT: v_mov_b32_e32 v2, s67 +; TONGA-NEXT: v_mov_b32_e32 v3, s51 +; TONGA-NEXT: s_addc_u32 s1, s37, 0 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: s_add_u32 s0, s36, 0x60 +; TONGA-NEXT: v_mov_b32_e32 v0, s30 +; TONGA-NEXT: v_mov_b32_e32 v1, s50 +; TONGA-NEXT: v_mov_b32_e32 v2, s31 +; TONGA-NEXT: v_mov_b32_e32 v3, s49 +; TONGA-NEXT: s_addc_u32 s1, s37, 0 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: s_add_u32 s0, s36, 0x50 +; TONGA-NEXT: v_mov_b32_e32 v0, s28 +; TONGA-NEXT: v_mov_b32_e32 v1, s39 +; TONGA-NEXT: v_mov_b32_e32 v2, s29 +; TONGA-NEXT: v_mov_b32_e32 v3, s38 +; TONGA-NEXT: s_addc_u32 s1, s37, 0 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: s_add_u32 s0, s36, 64 +; TONGA-NEXT: v_mov_b32_e32 v0, s26 +; TONGA-NEXT: v_mov_b32_e32 v1, s48 +; TONGA-NEXT: v_mov_b32_e32 v2, s27 +; TONGA-NEXT: v_mov_b32_e32 v3, s47 +; TONGA-NEXT: s_addc_u32 s1, s37, 0 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: s_add_u32 s0, s36, 48 +; TONGA-NEXT: v_mov_b32_e32 v0, s24 +; TONGA-NEXT: v_mov_b32_e32 v1, s46 +; TONGA-NEXT: v_mov_b32_e32 v2, s25 +; TONGA-NEXT: v_mov_b32_e32 v3, s45 +; TONGA-NEXT: s_addc_u32 s1, s37, 0 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: s_add_u32 s0, s36, 32 +; TONGA-NEXT: v_mov_b32_e32 v0, s22 +; TONGA-NEXT: v_mov_b32_e32 v1, s44 +; TONGA-NEXT: v_mov_b32_e32 v2, s23 +; TONGA-NEXT: v_mov_b32_e32 v3, s43 +; TONGA-NEXT: s_addc_u32 s1, s37, 0 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: s_add_u32 s0, s36, 16 +; TONGA-NEXT: v_mov_b32_e32 v0, s20 +; TONGA-NEXT: v_mov_b32_e32 v1, s42 +; TONGA-NEXT: v_mov_b32_e32 v2, s21 +; TONGA-NEXT: v_mov_b32_e32 v3, s41 +; TONGA-NEXT: s_addc_u32 s1, s37, 0 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: v_mov_b32_e32 v5, s1 +; TONGA-NEXT: v_mov_b32_e32 v0, s18 +; TONGA-NEXT: v_mov_b32_e32 v1, s40 +; TONGA-NEXT: v_mov_b32_e32 v2, s19 +; TONGA-NEXT: v_mov_b32_e32 v3, s35 +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: v_mov_b32_e32 v4, s36 +; TONGA-NEXT: v_mov_b32_e32 v0, s16 +; TONGA-NEXT: v_mov_b32_e32 v1, s34 +; TONGA-NEXT: v_mov_b32_e32 v2, s17 +; TONGA-NEXT: v_mov_b32_e32 v3, s33 +; TONGA-NEXT: v_mov_b32_e32 v5, s37 +; TONGA-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-NEXT: s_endpgm +; +; TONGA-GCNTRACKERS-LABEL: constant_zextload_v64i16_to_v64i32: +; TONGA-GCNTRACKERS: ; %bb.0: +; TONGA-GCNTRACKERS-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; TONGA-GCNTRACKERS-NEXT: s_load_dwordx16 s[0:15], s[38:39], 0x0 +; TONGA-GCNTRACKERS-NEXT: s_load_dwordx16 s[16:31], s[38:39], 0x40 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s33, s1, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s34, s0, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s35, s3, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s40, s2, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s41, s5, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s42, s4, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s43, s7, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s44, s6, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s45, s9, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s46, s8, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s47, s11, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s48, s10, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s38, s13, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s39, s12, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s49, s15, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s50, s14, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s51, s17, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s52, s16, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s53, s19, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s54, s18, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s55, s21, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s56, s20, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s57, s23, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s58, s22, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s59, s25, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s60, s24, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s61, s27, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s62, s26, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s63, s29, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s64, s28, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s65, s31, 16 +; TONGA-GCNTRACKERS-NEXT: s_lshr_b32 s66, s30, 16 +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s1, s1, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s0, s0, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s3, s3, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s2, s2, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s5, s5, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s4, s4, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s7, s7, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s6, s6, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s9, s9, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s8, s8, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s67, s11, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s68, s10, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s13, s13, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s12, s12, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s15, s15, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s14, s14, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s17, s17, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s16, s16, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s19, s19, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s18, s18, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s21, s21, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s20, s20, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s23, s23, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s22, s22, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s25, s25, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s24, s24, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s27, s27, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s26, s26, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s29, s29, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s28, s28, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s10, s31, 0xffff +; TONGA-GCNTRACKERS-NEXT: s_and_b32 s11, s30, 0xffff +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s10 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s10, s36, 0xf0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s11 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s11, s37, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s10 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s11 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s10, s36, 0xe0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s66 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s65 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s11, s37, 0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s10 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s11 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s10, s36, 0xd0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s28 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s64 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s29 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s63 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s11, s37, 0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s10 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s11 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s10, s36, 0xc0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s26 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s62 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s27 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s61 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s11, s37, 0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s10 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s11 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s10, s36, 0xb0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s24 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s60 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s25 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s59 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s11, s37, 0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s10 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s11 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s10, s36, 0xa0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s22 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s58 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s23 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s57 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s11, s37, 0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s10 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s11 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s10, s36, 0x90 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s20 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s56 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s21 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s55 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s11, s37, 0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s10 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s11 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s10, s36, 0x80 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s18 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s54 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s19 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s53 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s11, s37, 0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s10 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s11 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s10, s36, 0x70 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s16 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s52 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s17 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s51 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s11, s37, 0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s10 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s11 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s10, s36, 0x60 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s14 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s50 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s15 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s49 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s11, s37, 0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s10 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s11 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s10, s36, 0x50 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s12 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s39 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s13 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s38 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s11, s37, 0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s10 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s68 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s48 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s67 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s47 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s11 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: s_nop 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s8 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s8, s36, 64 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s9 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s9, s37, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s46 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s45 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s8 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s9 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: s_nop 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s6 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s6, s36, 48 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s7 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s7, s37, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s44 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s43 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s6 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s7 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: s_nop 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s4 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s4, s36, 32 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s5 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s5, s37, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s42 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s41 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s4 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s5 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: s_nop 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s2 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s2, s36, 16 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s3 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s3, s37, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s40 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s35 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s3 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s2 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s36 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s34 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s1 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s33 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s37 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx4 v[4:5], v[0:3] +; TONGA-GCNTRACKERS-NEXT: s_endpgm +; +; GFX908-LABEL: constant_zextload_v64i16_to_v64i32: +; GFX908: ; %bb.0: +; GFX908-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24 +; GFX908-NEXT: v_mov_b32_e32 v0, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_load_dwordx16 s[0:15], s[38:39], 0x40 +; GFX908-NEXT: s_load_dwordx16 s[16:31], s[38:39], 0x0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_lshr_b32 s65, s15, 16 +; GFX908-NEXT: s_lshr_b32 s66, s14, 16 +; GFX908-NEXT: s_and_b32 s15, s15, 0xffff +; GFX908-NEXT: s_and_b32 s14, s14, 0xffff +; GFX908-NEXT: s_lshr_b32 s63, s13, 16 +; GFX908-NEXT: s_lshr_b32 s64, s12, 16 +; GFX908-NEXT: s_and_b32 s13, s13, 0xffff +; GFX908-NEXT: s_and_b32 s12, s12, 0xffff +; GFX908-NEXT: v_mov_b32_e32 v1, s14 +; GFX908-NEXT: v_mov_b32_e32 v2, s66 +; GFX908-NEXT: v_mov_b32_e32 v3, s15 +; GFX908-NEXT: v_mov_b32_e32 v4, s65 +; GFX908-NEXT: s_lshr_b32 s61, s11, 16 +; GFX908-NEXT: s_lshr_b32 s62, s10, 16 +; GFX908-NEXT: s_and_b32 s11, s11, 0xffff +; GFX908-NEXT: s_and_b32 s10, s10, 0xffff +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:240 +; GFX908-NEXT: s_lshr_b32 s59, s9, 16 +; GFX908-NEXT: v_mov_b32_e32 v1, s12 +; GFX908-NEXT: v_mov_b32_e32 v2, s64 +; GFX908-NEXT: v_mov_b32_e32 v3, s13 +; GFX908-NEXT: v_mov_b32_e32 v4, s63 +; GFX908-NEXT: s_lshr_b32 s60, s8, 16 +; GFX908-NEXT: s_and_b32 s9, s9, 0xffff +; GFX908-NEXT: s_and_b32 s8, s8, 0xffff +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:224 +; GFX908-NEXT: s_lshr_b32 s57, s7, 16 +; GFX908-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-NEXT: v_mov_b32_e32 v2, s62 +; GFX908-NEXT: v_mov_b32_e32 v3, s11 +; GFX908-NEXT: v_mov_b32_e32 v4, s61 +; GFX908-NEXT: s_lshr_b32 s58, s6, 16 +; GFX908-NEXT: s_and_b32 s7, s7, 0xffff +; GFX908-NEXT: s_and_b32 s6, s6, 0xffff +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:208 +; GFX908-NEXT: s_lshr_b32 s55, s5, 16 +; GFX908-NEXT: v_mov_b32_e32 v1, s8 +; GFX908-NEXT: v_mov_b32_e32 v2, s60 +; GFX908-NEXT: v_mov_b32_e32 v3, s9 +; GFX908-NEXT: v_mov_b32_e32 v4, s59 +; GFX908-NEXT: s_lshr_b32 s56, s4, 16 +; GFX908-NEXT: s_and_b32 s5, s5, 0xffff +; GFX908-NEXT: s_and_b32 s4, s4, 0xffff +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:192 +; GFX908-NEXT: s_lshr_b32 s53, s3, 16 +; GFX908-NEXT: v_mov_b32_e32 v1, s6 +; GFX908-NEXT: v_mov_b32_e32 v2, s58 +; GFX908-NEXT: v_mov_b32_e32 v3, s7 +; GFX908-NEXT: v_mov_b32_e32 v4, s57 +; GFX908-NEXT: s_lshr_b32 s54, s2, 16 +; GFX908-NEXT: s_and_b32 s3, s3, 0xffff +; GFX908-NEXT: s_and_b32 s2, s2, 0xffff +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:176 +; GFX908-NEXT: s_lshr_b32 s51, s1, 16 +; GFX908-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-NEXT: v_mov_b32_e32 v2, s56 +; GFX908-NEXT: v_mov_b32_e32 v3, s5 +; GFX908-NEXT: v_mov_b32_e32 v4, s55 +; GFX908-NEXT: s_lshr_b32 s52, s0, 16 +; GFX908-NEXT: s_and_b32 s1, s1, 0xffff +; GFX908-NEXT: s_and_b32 s0, s0, 0xffff +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:160 +; GFX908-NEXT: s_lshr_b32 s49, s31, 16 +; GFX908-NEXT: v_mov_b32_e32 v1, s2 +; GFX908-NEXT: v_mov_b32_e32 v2, s54 +; GFX908-NEXT: v_mov_b32_e32 v3, s3 +; GFX908-NEXT: v_mov_b32_e32 v4, s53 +; GFX908-NEXT: s_lshr_b32 s50, s30, 16 +; GFX908-NEXT: s_and_b32 s31, s31, 0xffff +; GFX908-NEXT: s_and_b32 s30, s30, 0xffff +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:144 +; GFX908-NEXT: s_lshr_b32 s38, s29, 16 +; GFX908-NEXT: v_mov_b32_e32 v1, s0 +; GFX908-NEXT: v_mov_b32_e32 v2, s52 +; GFX908-NEXT: v_mov_b32_e32 v3, s1 +; GFX908-NEXT: v_mov_b32_e32 v4, s51 +; GFX908-NEXT: s_lshr_b32 s39, s28, 16 +; GFX908-NEXT: s_and_b32 s29, s29, 0xffff +; GFX908-NEXT: s_and_b32 s28, s28, 0xffff +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:128 +; GFX908-NEXT: s_lshr_b32 s47, s27, 16 +; GFX908-NEXT: v_mov_b32_e32 v1, s30 +; GFX908-NEXT: v_mov_b32_e32 v2, s50 +; GFX908-NEXT: v_mov_b32_e32 v3, s31 +; GFX908-NEXT: v_mov_b32_e32 v4, s49 +; GFX908-NEXT: s_lshr_b32 s48, s26, 16 +; GFX908-NEXT: s_and_b32 s27, s27, 0xffff +; GFX908-NEXT: s_and_b32 s26, s26, 0xffff +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:112 +; GFX908-NEXT: s_lshr_b32 s45, s25, 16 +; GFX908-NEXT: v_mov_b32_e32 v1, s28 +; GFX908-NEXT: v_mov_b32_e32 v2, s39 +; GFX908-NEXT: v_mov_b32_e32 v3, s29 +; GFX908-NEXT: v_mov_b32_e32 v4, s38 +; GFX908-NEXT: s_lshr_b32 s46, s24, 16 +; GFX908-NEXT: s_and_b32 s25, s25, 0xffff +; GFX908-NEXT: s_and_b32 s24, s24, 0xffff +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:96 +; GFX908-NEXT: s_lshr_b32 s43, s23, 16 +; GFX908-NEXT: v_mov_b32_e32 v1, s26 +; GFX908-NEXT: v_mov_b32_e32 v2, s48 +; GFX908-NEXT: v_mov_b32_e32 v3, s27 +; GFX908-NEXT: v_mov_b32_e32 v4, s47 +; GFX908-NEXT: s_lshr_b32 s44, s22, 16 +; GFX908-NEXT: s_and_b32 s23, s23, 0xffff +; GFX908-NEXT: s_and_b32 s22, s22, 0xffff +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:80 +; GFX908-NEXT: s_lshr_b32 s41, s21, 16 +; GFX908-NEXT: v_mov_b32_e32 v1, s24 +; GFX908-NEXT: v_mov_b32_e32 v2, s46 +; GFX908-NEXT: v_mov_b32_e32 v3, s25 +; GFX908-NEXT: v_mov_b32_e32 v4, s45 +; GFX908-NEXT: s_lshr_b32 s42, s20, 16 +; GFX908-NEXT: s_and_b32 s21, s21, 0xffff +; GFX908-NEXT: s_and_b32 s20, s20, 0xffff +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:64 +; GFX908-NEXT: s_lshr_b32 s35, s19, 16 +; GFX908-NEXT: v_mov_b32_e32 v1, s22 +; GFX908-NEXT: v_mov_b32_e32 v2, s44 +; GFX908-NEXT: v_mov_b32_e32 v3, s23 +; GFX908-NEXT: v_mov_b32_e32 v4, s43 +; GFX908-NEXT: s_lshr_b32 s40, s18, 16 +; GFX908-NEXT: s_and_b32 s19, s19, 0xffff +; GFX908-NEXT: s_and_b32 s18, s18, 0xffff +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:48 +; GFX908-NEXT: s_lshr_b32 s33, s17, 16 +; GFX908-NEXT: v_mov_b32_e32 v1, s20 +; GFX908-NEXT: v_mov_b32_e32 v2, s42 +; GFX908-NEXT: v_mov_b32_e32 v3, s21 +; GFX908-NEXT: v_mov_b32_e32 v4, s41 +; GFX908-NEXT: s_lshr_b32 s34, s16, 16 +; GFX908-NEXT: s_and_b32 s17, s17, 0xffff +; GFX908-NEXT: s_and_b32 s16, s16, 0xffff +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:32 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_mov_b32_e32 v1, s18 +; GFX908-NEXT: v_mov_b32_e32 v2, s40 +; GFX908-NEXT: v_mov_b32_e32 v3, s19 +; GFX908-NEXT: v_mov_b32_e32 v4, s35 +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:16 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_mov_b32_e32 v1, s16 +; GFX908-NEXT: v_mov_b32_e32 v2, s34 +; GFX908-NEXT: v_mov_b32_e32 v3, s17 +; GFX908-NEXT: v_mov_b32_e32 v4, s33 +; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] +; GFX908-NEXT: s_endpgm +; +; GFX908-GCNTRACKERS-LABEL: constant_zextload_v64i16_to_v64i32: +; GFX908-GCNTRACKERS: ; %bb.0: +; GFX908-GCNTRACKERS-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, 0 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GCNTRACKERS-NEXT: s_load_dwordx16 s[16:31], s[38:39], 0x40 +; GFX908-GCNTRACKERS-NEXT: s_load_dwordx16 s[0:15], s[38:39], 0x0 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s65, s31, 16 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s66, s30, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s31, s31, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s30, s30, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s63, s29, 16 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s64, s28, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s29, s29, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s28, s28, 0xffff +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s30 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s66 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s31 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s65 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s61, s27, 16 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s62, s26, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s27, s27, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s26, s26, 0xffff +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:240 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s59, s25, 16 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s28 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s64 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s29 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s63 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s60, s24, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s25, s25, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s24, s24, 0xffff +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:224 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s57, s23, 16 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s26 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s62 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s27 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s61 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s58, s22, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s23, s23, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s22, s22, 0xffff +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:208 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s55, s21, 16 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s24 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s60 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s25 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s59 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s56, s20, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s21, s21, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s20, s20, 0xffff +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:192 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s53, s19, 16 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s22 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s58 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s23 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s57 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s54, s18, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s19, s19, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s18, s18, 0xffff +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:176 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s51, s17, 16 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s20 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s56 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s21 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s55 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s52, s16, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s17, s17, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s16, s16, 0xffff +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:160 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s49, s15, 16 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s18 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s54 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s19 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s53 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s50, s14, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s15, s15, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s14, s14, 0xffff +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:144 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s38, s13, 16 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s16 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s52 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s17 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s51 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s39, s12, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s13, s13, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s12, s12, 0xffff +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:128 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s47, s11, 16 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s14 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s50 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s15 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s49 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s48, s10, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s11, s11, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s10, s10, 0xffff +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:112 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s45, s9, 16 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s12 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s39 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s13 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s38 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s46, s8, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s9, s9, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s8, s8, 0xffff +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:96 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s43, s7, 16 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s48 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s11 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s47 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s44, s6, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s7, s7, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s6, s6, 0xffff +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:80 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s41, s5, 16 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s8 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s46 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s9 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s45 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s42, s4, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s5, s5, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s4, s4, 0xffff +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:64 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s35, s3, 16 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s6 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s44 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s7 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s43 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s40, s2, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s3, s3, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s2, s2, 0xffff +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:48 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s33, s1, 16 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s42 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s5 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s41 +; GFX908-GCNTRACKERS-NEXT: s_lshr_b32 s34, s0, 16 +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s1, s1, 0xffff +; GFX908-GCNTRACKERS-NEXT: s_and_b32 s0, s0, 0xffff +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:32 +; GFX908-GCNTRACKERS-NEXT: s_nop 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s2 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s40 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s3 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s35 +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] offset:16 +; GFX908-GCNTRACKERS-NEXT: s_nop 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s34 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s1 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s33 +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx4 v0, v[1:4], s[36:37] +; GFX908-GCNTRACKERS-NEXT: s_endpgm +; +; GENERIC-LABEL: constant_zextload_v64i16_to_v64i32: +; GENERIC: ; %bb.0: +; GENERIC-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x9 +; GENERIC-NEXT: s_waitcnt lgkmcnt(0) +; GENERIC-NEXT: s_load_dwordx16 s[0:15], s[38:39], 0x0 +; GENERIC-NEXT: s_load_dwordx16 s[16:31], s[38:39], 0x10 +; GENERIC-NEXT: s_waitcnt lgkmcnt(0) +; GENERIC-NEXT: s_lshr_b32 s35, s3, 16 +; GENERIC-NEXT: s_lshr_b32 s40, s2, 16 +; GENERIC-NEXT: s_and_b32 s55, s3, 0xffff +; GENERIC-NEXT: s_and_b32 s56, s2, 0xffff +; GENERIC-NEXT: s_lshr_b32 s2, s22, 16 +; GENERIC-NEXT: s_lshr_b32 s3, s25, 16 +; GENERIC-NEXT: s_lshr_b32 s67, s31, 16 +; GENERIC-NEXT: s_lshr_b32 s68, s30, 16 +; GENERIC-NEXT: s_and_b32 s31, s31, 0xffff +; GENERIC-NEXT: s_and_b32 s30, s30, 0xffff +; GENERIC-NEXT: s_lshr_b32 s33, s1, 16 +; GENERIC-NEXT: s_lshr_b32 s34, s0, 16 +; GENERIC-NEXT: s_and_b32 s44, s1, 0xffff +; GENERIC-NEXT: s_and_b32 s46, s0, 0xffff +; GENERIC-NEXT: s_lshr_b32 s59, s21, 16 +; GENERIC-NEXT: s_lshr_b32 s60, s20, 16 +; GENERIC-NEXT: s_lshr_b32 s65, s29, 16 +; GENERIC-NEXT: s_lshr_b32 s66, s28, 16 +; GENERIC-NEXT: s_and_b32 s21, s21, 0xffff +; GENERIC-NEXT: s_and_b32 s20, s20, 0xffff +; GENERIC-NEXT: s_and_b32 s29, s29, 0xffff +; GENERIC-NEXT: s_and_b32 s28, s28, 0xffff +; GENERIC-NEXT: s_mov_b32 s0, s36 +; GENERIC-NEXT: s_mov_b32 s1, s37 +; GENERIC-NEXT: v_mov_b32_e32 v0, s30 +; GENERIC-NEXT: v_mov_b32_e32 v1, s68 +; GENERIC-NEXT: v_mov_b32_e32 v2, s31 +; GENERIC-NEXT: v_mov_b32_e32 v3, s67 +; GENERIC-NEXT: v_mov_b32_e32 v15, s3 +; GENERIC-NEXT: v_mov_b32_e32 v17, s2 +; GENERIC-NEXT: s_mov_b32 s3, 0xf000 +; GENERIC-NEXT: s_mov_b32 s2, -1 +; GENERIC-NEXT: s_lshr_b32 s57, s19, 16 +; GENERIC-NEXT: s_lshr_b32 s58, s18, 16 +; GENERIC-NEXT: s_lshr_b32 s61, s23, 16 +; GENERIC-NEXT: s_lshr_b32 s62, s24, 16 +; GENERIC-NEXT: s_lshr_b32 s63, s27, 16 +; GENERIC-NEXT: s_lshr_b32 s64, s26, 16 +; GENERIC-NEXT: s_and_b32 s19, s19, 0xffff +; GENERIC-NEXT: s_and_b32 s18, s18, 0xffff +; GENERIC-NEXT: s_and_b32 s23, s23, 0xffff +; GENERIC-NEXT: s_and_b32 s22, s22, 0xffff +; GENERIC-NEXT: s_and_b32 s25, s25, 0xffff +; GENERIC-NEXT: s_and_b32 s24, s24, 0xffff +; GENERIC-NEXT: s_and_b32 s27, s27, 0xffff +; GENERIC-NEXT: s_and_b32 s26, s26, 0xffff +; GENERIC-NEXT: v_mov_b32_e32 v4, s28 +; GENERIC-NEXT: v_mov_b32_e32 v5, s66 +; GENERIC-NEXT: v_mov_b32_e32 v6, s29 +; GENERIC-NEXT: v_mov_b32_e32 v7, s65 +; GENERIC-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:240 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_mov_b32_e32 v0, s20 +; GENERIC-NEXT: v_mov_b32_e32 v1, s60 +; GENERIC-NEXT: v_mov_b32_e32 v2, s21 +; GENERIC-NEXT: v_mov_b32_e32 v3, s59 +; GENERIC-NEXT: s_lshr_b32 s38, s17, 16 +; GENERIC-NEXT: s_lshr_b32 s39, s16, 16 +; GENERIC-NEXT: s_and_b32 s17, s17, 0xffff +; GENERIC-NEXT: s_and_b32 s16, s16, 0xffff +; GENERIC-NEXT: v_mov_b32_e32 v8, s26 +; GENERIC-NEXT: v_mov_b32_e32 v9, s64 +; GENERIC-NEXT: v_mov_b32_e32 v10, s27 +; GENERIC-NEXT: v_mov_b32_e32 v11, s63 +; GENERIC-NEXT: v_mov_b32_e32 v12, s24 +; GENERIC-NEXT: v_mov_b32_e32 v13, s62 +; GENERIC-NEXT: v_mov_b32_e32 v14, s25 +; GENERIC-NEXT: v_mov_b32_e32 v16, s22 +; GENERIC-NEXT: v_mov_b32_e32 v18, s23 +; GENERIC-NEXT: v_mov_b32_e32 v19, s61 +; GENERIC-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:224 +; GENERIC-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:208 +; GENERIC-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:192 +; GENERIC-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:176 +; GENERIC-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:160 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_mov_b32_e32 v0, s18 +; GENERIC-NEXT: v_mov_b32_e32 v1, s58 +; GENERIC-NEXT: v_mov_b32_e32 v2, s19 +; GENERIC-NEXT: v_mov_b32_e32 v3, s57 +; GENERIC-NEXT: s_lshr_b32 s53, s15, 16 +; GENERIC-NEXT: s_lshr_b32 s54, s14, 16 +; GENERIC-NEXT: s_and_b32 s15, s15, 0xffff +; GENERIC-NEXT: s_and_b32 s14, s14, 0xffff +; GENERIC-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:144 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_mov_b32_e32 v0, s16 +; GENERIC-NEXT: v_mov_b32_e32 v1, s39 +; GENERIC-NEXT: v_mov_b32_e32 v2, s17 +; GENERIC-NEXT: v_mov_b32_e32 v3, s38 +; GENERIC-NEXT: s_lshr_b32 s51, s13, 16 +; GENERIC-NEXT: s_lshr_b32 s52, s12, 16 +; GENERIC-NEXT: s_and_b32 s13, s13, 0xffff +; GENERIC-NEXT: s_and_b32 s12, s12, 0xffff +; GENERIC-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:128 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_mov_b32_e32 v0, s14 +; GENERIC-NEXT: v_mov_b32_e32 v1, s54 +; GENERIC-NEXT: v_mov_b32_e32 v2, s15 +; GENERIC-NEXT: v_mov_b32_e32 v3, s53 +; GENERIC-NEXT: s_lshr_b32 s49, s11, 16 +; GENERIC-NEXT: s_lshr_b32 s50, s10, 16 +; GENERIC-NEXT: s_and_b32 s11, s11, 0xffff +; GENERIC-NEXT: s_and_b32 s10, s10, 0xffff +; GENERIC-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:112 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_mov_b32_e32 v0, s12 +; GENERIC-NEXT: v_mov_b32_e32 v1, s52 +; GENERIC-NEXT: v_mov_b32_e32 v2, s13 +; GENERIC-NEXT: v_mov_b32_e32 v3, s51 +; GENERIC-NEXT: s_lshr_b32 s47, s9, 16 +; GENERIC-NEXT: s_lshr_b32 s48, s8, 16 +; GENERIC-NEXT: s_and_b32 s9, s9, 0xffff +; GENERIC-NEXT: s_and_b32 s8, s8, 0xffff +; GENERIC-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:96 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_mov_b32_e32 v0, s10 +; GENERIC-NEXT: v_mov_b32_e32 v1, s50 +; GENERIC-NEXT: v_mov_b32_e32 v2, s11 +; GENERIC-NEXT: v_mov_b32_e32 v3, s49 +; GENERIC-NEXT: s_lshr_b32 s43, s7, 16 +; GENERIC-NEXT: s_lshr_b32 s45, s6, 16 +; GENERIC-NEXT: s_and_b32 s7, s7, 0xffff +; GENERIC-NEXT: s_and_b32 s6, s6, 0xffff +; GENERIC-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:80 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_mov_b32_e32 v0, s8 +; GENERIC-NEXT: v_mov_b32_e32 v1, s48 +; GENERIC-NEXT: v_mov_b32_e32 v2, s9 +; GENERIC-NEXT: v_mov_b32_e32 v3, s47 +; GENERIC-NEXT: s_lshr_b32 s41, s5, 16 +; GENERIC-NEXT: s_lshr_b32 s42, s4, 16 +; GENERIC-NEXT: s_and_b32 s5, s5, 0xffff +; GENERIC-NEXT: s_and_b32 s4, s4, 0xffff +; GENERIC-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_mov_b32_e32 v0, s6 +; GENERIC-NEXT: v_mov_b32_e32 v1, s45 +; GENERIC-NEXT: v_mov_b32_e32 v2, s7 +; GENERIC-NEXT: v_mov_b32_e32 v3, s43 +; GENERIC-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_mov_b32_e32 v0, s4 +; GENERIC-NEXT: v_mov_b32_e32 v1, s42 +; GENERIC-NEXT: v_mov_b32_e32 v2, s5 +; GENERIC-NEXT: v_mov_b32_e32 v3, s41 +; GENERIC-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_mov_b32_e32 v0, s56 +; GENERIC-NEXT: v_mov_b32_e32 v1, s40 +; GENERIC-NEXT: v_mov_b32_e32 v2, s55 +; GENERIC-NEXT: v_mov_b32_e32 v3, s35 +; GENERIC-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_mov_b32_e32 v0, s46 +; GENERIC-NEXT: v_mov_b32_e32 v1, s34 +; GENERIC-NEXT: v_mov_b32_e32 v2, s44 +; GENERIC-NEXT: v_mov_b32_e32 v3, s33 +; GENERIC-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 +; GENERIC-NEXT: s_endpgm +; +; GENERIC-GCNTRACKERS-LABEL: constant_zextload_v64i16_to_v64i32: +; GENERIC-GCNTRACKERS: ; %bb.0: +; GENERIC-GCNTRACKERS-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x9 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s39, 0xf000 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s38, -1 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: s_load_dwordx16 s[0:15], s[18:19], 0x0 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s36, s16 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s37, s17 +; GENERIC-GCNTRACKERS-NEXT: s_load_dwordx16 s[16:31], s[18:19], 0x10 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s43, s7, 16 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s7, s7, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s41, s5, 16 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s44, s30, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s30, s30, 16 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s30 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s30, s31, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s31, s31, 16 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s31 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s31, s28, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s28, s28, 16 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s28 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s28, s29, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s29, s29, 16 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v7, s29 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s29, s26, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s26, s26, 16 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v9, s26 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s26, s27, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s27, s27, 16 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v11, s27 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s27, s24, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s24, s24, 16 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v13, s24 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s24, s25, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s25, s25, 16 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v15, s25 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s25, s22, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s22, s22, 16 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v17, s22 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s22, s23, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s23, s23, 16 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s44 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s30 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v19, s23 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s23, s20, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s20, s20, 16 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:240 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s20 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s20, s21, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s21, s21, 16 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s31 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v6, s28 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s23 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s20 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s20, s19, 16 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s21 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s21, s18, 16 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s19, s19, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s18, s18, 0xffff +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v8, s29 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v10, s26 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v12, s27 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v14, s24 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v16, s25 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v18, s22 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s22, s17, 16 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s23, s16, 16 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s17, s17, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s16, s16, 0xffff +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[4:7], off, s[36:39], 0 offset:224 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[8:11], off, s[36:39], 0 offset:208 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[12:15], off, s[36:39], 0 offset:192 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[16:19], off, s[36:39], 0 offset:176 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:160 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s18 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s21 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s19 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s20 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s24, s15, 16 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s25, s14, 16 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s15, s15, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s14, s14, 0xffff +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:144 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s16 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s23 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s17 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s22 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s26, s13, 16 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s27, s12, 16 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s13, s13, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s12, s12, 0xffff +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:128 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s14 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s25 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s15 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s24 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s28, s11, 16 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s29, s10, 16 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s11, s11, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s10, s10, 0xffff +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:112 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s12 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s27 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s13 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s26 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s30, s9, 16 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s31, s8, 16 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s9, s9, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s8, s8, 0xffff +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:96 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s10 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s29 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s11 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s28 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s44, s6, 16 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s6, s6, 0xffff +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:80 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s8 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s31 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s9 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s30 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s42, s4, 16 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s5, s5, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s4, s4, 0xffff +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:64 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s6 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s44 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s7 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s43 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s35, s3, 16 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s40, s2, 16 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s3, s3, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s2, s2, 0xffff +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:48 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s4 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s42 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s5 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s41 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s33, s1, 16 +; GENERIC-GCNTRACKERS-NEXT: s_lshr_b32 s34, s0, 16 +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s1, s1, 0xffff +; GENERIC-GCNTRACKERS-NEXT: s_and_b32 s0, s0, 0xffff +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:32 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s2 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s40 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s3 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s35 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 offset:16 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s34 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s1 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s33 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx4 v[0:3], off, s[36:39], 0 +; GENERIC-GCNTRACKERS-NEXT: s_endpgm %load = load <64 x i16>, ptr addrspace(4) %in %ext = zext <64 x i16> %load to <64 x i32> store <64 x i32> %ext, ptr addrspace(1) %out @@ -82,6 +5482,2986 @@ define protected amdgpu_kernel void @excess_soft_clause_reg_pressure(ptr addrspace(4) %wei_ptr, ptr addrspace(1) %out_ptr, ptr addrspace(1) %in) { +; GFX11-PAL-LABEL: excess_soft_clause_reg_pressure: +; GFX11-PAL: ; %bb.0: ; %entry +; GFX11-PAL-NEXT: s_load_b256 s[0:7], s[4:5], 0x0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v11, 0 :: v_dual_and_b32 v34, 0x3ff, v0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v6, 0 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-PAL-NEXT: v_dual_mov_b32 v21, 0 :: v_dual_lshlrev_b32 v2, 2, v34 +; GFX11-PAL-NEXT: v_dual_mov_b32 v7, 0 :: v_dual_mov_b32 v8, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v10, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v17, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v19, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v18, 0 :: v_dual_mov_b32 v23, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v20, 0 :: v_dual_mov_b32 v25, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v22, 0 :: v_dual_mov_b32 v27, 0 +; GFX11-PAL-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PAL-NEXT: v_add_co_u32 v2, s4, s4, v2 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v3, null, s5, 0, s4 +; GFX11-PAL-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v29, 0 +; GFX11-PAL-NEXT: v_add_co_u32 v4, vcc_lo, 0x188, v2 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v3, vcc_lo +; GFX11-PAL-NEXT: v_dual_mov_b32 v26, 0 :: v_dual_mov_b32 v3, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v28, 0 :: v_dual_mov_b32 v31, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v33, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v30, 0 :: v_dual_mov_b32 v13, 0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v32, 0 :: v_dual_mov_b32 v15, 0 +; GFX11-PAL-NEXT: v_mov_b32_e32 v12, 0 +; GFX11-PAL-NEXT: v_mov_b32_e32 v14, 0 +; GFX11-PAL-NEXT: s_movk_i32 s4, 0x200 +; GFX11-PAL-NEXT: .LBB4_1: ; %for.cond28.preheader +; GFX11-PAL-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-PAL-NEXT: s_clause 0x3 +; GFX11-PAL-NEXT: global_load_b32 v35, v[4:5], off offset:-392 +; GFX11-PAL-NEXT: global_load_b32 v36, v[4:5], off offset:-196 +; GFX11-PAL-NEXT: global_load_b32 v37, v[4:5], off +; GFX11-PAL-NEXT: global_load_b32 v38, v[4:5], off offset:196 +; GFX11-PAL-NEXT: s_clause 0x4 +; GFX11-PAL-NEXT: s_load_b512 s[84:99], s[0:1], 0x0 +; GFX11-PAL-NEXT: s_load_b512 s[52:67], s[0:1], 0x1000 +; GFX11-PAL-NEXT: s_load_b512 s[68:83], s[0:1], 0x40 +; GFX11-PAL-NEXT: s_load_b512 s[36:51], s[0:1], 0x2000 +; GFX11-PAL-NEXT: s_load_b512 s[16:31], s[0:1], 0x3000 +; GFX11-PAL-NEXT: v_add_co_u32 v4, vcc_lo, 0x310, v4 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo +; GFX11-PAL-NEXT: s_waitcnt vmcnt(3) lgkmcnt(0) +; GFX11-PAL-NEXT: v_fmac_f32_e32 v32, s85, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v33, s84, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v30, s87, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v31, s86, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v3, s89, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v2, s88, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v28, s92, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v1, s90, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v26, s94, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v29, s91, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v24, s96, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v27, s93, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v22, s98, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v25, s95, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v20, s68, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v23, s97, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v18, s70, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v21, s99, v35 +; GFX11-PAL-NEXT: s_load_b512 s[84:99], s[0:1], 0x1040 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v16, s72, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v19, s69, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v0, s74, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v17, s71, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v10, s75, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v11, s73, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v8, s77, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v9, s76, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v6, s79, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v7, s78, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v13, s81, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v12, s80, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v15, s83, v35 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v14, s82, v35 +; GFX11-PAL-NEXT: s_load_b512 s[68:83], s[0:1], 0x2040 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(2) +; GFX11-PAL-NEXT: v_fmac_f32_e32 v32, s53, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v33, s52, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v30, s55, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v31, s54, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v3, s57, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v2, s56, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v28, s60, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v1, s58, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v26, s62, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v29, s59, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v24, s64, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v27, s61, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v22, s66, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v25, s63, v36 +; GFX11-PAL-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PAL-NEXT: v_fmac_f32_e32 v20, s84, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v23, s65, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v18, s86, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v21, s67, v36 +; GFX11-PAL-NEXT: s_load_b512 s[52:67], s[0:1], 0x3040 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v16, s88, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v19, s85, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v0, s90, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v17, s87, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v10, s91, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v11, s89, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v8, s93, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v9, s92, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v6, s95, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v7, s94, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v13, s97, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v12, s96, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v15, s99, v36 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v14, s98, v36 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(1) +; GFX11-PAL-NEXT: v_fmac_f32_e32 v32, s37, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v33, s36, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v30, s39, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v31, s38, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v3, s41, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v2, s40, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v28, s44, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v1, s42, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v26, s46, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v29, s43, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v24, s48, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v27, s45, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v22, s50, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v25, s47, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v20, s68, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v23, s49, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v18, s70, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v21, s51, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v16, s72, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v19, s69, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v0, s74, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v17, s71, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v10, s75, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v11, s73, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v8, s77, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v9, s76, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v6, s79, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v7, s78, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v13, s81, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v12, s80, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v15, s83, v37 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v14, s82, v37 +; GFX11-PAL-NEXT: s_add_u32 s0, s0, 0x4000 +; GFX11-PAL-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-NEXT: v_fmac_f32_e32 v32, s17, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v33, s16, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v30, s19, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v31, s18, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v3, s21, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v2, s20, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v28, s24, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v1, s22, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v26, s26, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v29, s23, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v24, s28, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v27, s25, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v22, s30, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v25, s27, v38 +; GFX11-PAL-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PAL-NEXT: v_fmac_f32_e32 v20, s52, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v23, s29, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v18, s54, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v21, s31, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v16, s56, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v19, s53, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v0, s58, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v17, s55, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v10, s59, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v11, s57, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v8, s61, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v9, s60, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v6, s63, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v7, s62, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v13, s65, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v12, s64, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v15, s67, v38 +; GFX11-PAL-NEXT: v_fmac_f32_e32 v14, s66, v38 +; GFX11-PAL-NEXT: s_addc_u32 s1, s1, 0 +; GFX11-PAL-NEXT: s_add_i32 s4, s4, -1 +; GFX11-PAL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-PAL-NEXT: s_cmp_lg_u32 s4, 0 +; GFX11-PAL-NEXT: s_cbranch_scc1 .LBB4_1 +; GFX11-PAL-NEXT: ; %bb.2: ; %for.cond.cleanup26 +; GFX11-PAL-NEXT: s_lshl_b32 s0, s13, 8 +; GFX11-PAL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1) +; GFX11-PAL-NEXT: v_add3_u32 v4, s6, s0, v34 +; GFX11-PAL-NEXT: s_mov_b32 s0, 0 +; GFX11-PAL-NEXT: v_mov_b32_e32 v5, 0 +; GFX11-PAL-NEXT: s_mov_b32 s1, s0 +; GFX11-PAL-NEXT: v_dual_mov_b32 v36, s0 :: v_dual_mov_b32 v37, s1 +; GFX11-PAL-NEXT: v_mul_hi_u32 v4, 0x5397829d, v4 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-PAL-NEXT: v_lshrrev_b32_e32 v4, 4, v4 +; GFX11-PAL-NEXT: v_mul_lo_u32 v4, 0x31000, v4 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-PAL-NEXT: v_lshlrev_b64 v[34:35], 2, v[4:5] +; GFX11-PAL-NEXT: v_mov_b32_e32 v4, s0 +; GFX11-PAL-NEXT: v_add_co_u32 v34, vcc_lo, s2, v34 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v35, null, s3, v35, vcc_lo +; GFX11-PAL-NEXT: s_clause 0x5 +; GFX11-PAL-NEXT: global_store_b32 v[34:35], v33, off +; GFX11-PAL-NEXT: global_store_b32 v[34:35], v32, off offset:784 +; GFX11-PAL-NEXT: global_store_b64 v[34:35], v[36:37], off offset:840 +; GFX11-PAL-NEXT: global_store_b32 v[34:35], v31, off offset:1568 +; GFX11-PAL-NEXT: global_store_b32 v[34:35], v30, off offset:2352 +; GFX11-PAL-NEXT: global_store_b32 v[34:35], v2, off offset:3136 +; GFX11-PAL-NEXT: v_add_co_u32 v38, vcc_lo, 0x310, v34 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v39, null, 0, v35, vcc_lo +; GFX11-PAL-NEXT: v_add_co_u32 v40, vcc_lo, 0x620, v34 +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v41, null, 0, v35, vcc_lo +; GFX11-PAL-NEXT: v_add_co_u32 v42, vcc_lo, 0x930, v34 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v43, null, 0, v35, vcc_lo +; GFX11-PAL-NEXT: v_add_co_u32 v44, vcc_lo, 0xc40, v34 +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v45, null, 0, v35, vcc_lo +; GFX11-PAL-NEXT: v_add_co_u32 v30, vcc_lo, 0xf50, v34 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v31, null, 0, v35, vcc_lo +; GFX11-PAL-NEXT: v_add_co_u32 v32, vcc_lo, 0xf50, v38 +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v33, null, 0, v39, vcc_lo +; GFX11-PAL-NEXT: v_mov_b32_e32 v2, s0 +; GFX11-PAL-NEXT: v_add_co_u32 v46, vcc_lo, 0xf50, v40 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v47, null, 0, v41, vcc_lo +; GFX11-PAL-NEXT: v_add_co_u32 v48, vcc_lo, 0xf50, v42 +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v43, vcc_lo +; GFX11-PAL-NEXT: s_clause 0x3 +; GFX11-PAL-NEXT: global_store_b64 v[34:35], v[3:4], off offset:3920 +; GFX11-PAL-NEXT: global_store_b64 v[38:39], v[1:2], off offset:3920 +; GFX11-PAL-NEXT: global_store_b32 v[40:41], v29, off offset:3920 +; GFX11-PAL-NEXT: global_store_b32 v[42:43], v28, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_u32 v1, vcc_lo, 0xf50, v44 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v2, null, 0, v45, vcc_lo +; GFX11-PAL-NEXT: v_add_co_u32 v3, vcc_lo, 0xf50, v30 +; GFX11-PAL-NEXT: s_clause 0x1 +; GFX11-PAL-NEXT: global_store_b64 v[42:43], v[36:37], off offset:3976 +; GFX11-PAL-NEXT: global_store_b32 v[44:45], v27, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v31, vcc_lo +; GFX11-PAL-NEXT: global_store_b32 v[30:31], v26, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_u32 v26, vcc_lo, 0xf50, v32 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v27, null, 0, v33, vcc_lo +; GFX11-PAL-NEXT: v_add_co_u32 v28, vcc_lo, 0xf50, v46 +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v29, null, 0, v47, vcc_lo +; GFX11-PAL-NEXT: global_store_b32 v[46:47], v24, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_u32 v24, vcc_lo, 0xf50, v48 +; GFX11-PAL-NEXT: global_store_b32 v[32:33], v25, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v25, null, 0, v49, vcc_lo +; GFX11-PAL-NEXT: v_add_co_u32 v30, vcc_lo, 0xf50, v1 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v31, null, 0, v2, vcc_lo +; GFX11-PAL-NEXT: global_store_b32 v[1:2], v22, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_u32 v1, vcc_lo, 0xf50, v3 +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v2, null, 0, v4, vcc_lo +; GFX11-PAL-NEXT: global_store_b32 v[3:4], v21, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_u32 v3, vcc_lo, 0xf50, v26 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v27, vcc_lo +; GFX11-PAL-NEXT: global_store_b32 v[26:27], v20, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_u32 v20, vcc_lo, 0xf50, v28 +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v21, null, 0, v29, vcc_lo +; GFX11-PAL-NEXT: v_add_co_u32 v22, vcc_lo, 0xf50, v24 +; GFX11-PAL-NEXT: global_store_b32 v[48:49], v23, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v23, null, 0, v25, vcc_lo +; GFX11-PAL-NEXT: global_store_b32 v[24:25], v18, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_u32 v18, vcc_lo, 0xf50, v30 +; GFX11-PAL-NEXT: global_store_b32 v[28:29], v19, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v19, null, 0, v31, vcc_lo +; GFX11-PAL-NEXT: v_add_co_u32 v24, vcc_lo, 0xf50, v1 +; GFX11-PAL-NEXT: global_store_b32 v[30:31], v17, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v25, null, 0, v2, vcc_lo +; GFX11-PAL-NEXT: global_store_b32 v[1:2], v16, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_u32 v1, vcc_lo, 0xf50, v3 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v2, null, 0, v4, vcc_lo +; GFX11-PAL-NEXT: global_store_b32 v[3:4], v11, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_u32 v3, vcc_lo, 0xf50, v20 +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v21, vcc_lo +; GFX11-PAL-NEXT: v_add_co_u32 v16, vcc_lo, 0xf50, v22 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v17, null, 0, v23, vcc_lo +; GFX11-PAL-NEXT: global_store_b32 v[22:23], v10, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_u32 v10, vcc_lo, 0xf50, v18 +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v19, vcc_lo +; GFX11-PAL-NEXT: global_store_b32 v[18:19], v9, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_u32 v18, vcc_lo, 0xf50, v24 +; GFX11-PAL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v19, null, 0, v25, vcc_lo +; GFX11-PAL-NEXT: global_store_b32 v[24:25], v8, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_u32 v8, vcc_lo, 0xf50, v1 +; GFX11-PAL-NEXT: global_store_b32 v[20:21], v0, off offset:3920 +; GFX11-PAL-NEXT: v_add_co_ci_u32_e64 v9, null, 0, v2, vcc_lo +; GFX11-PAL-NEXT: global_store_b32 v[1:2], v7, off offset:3920 +; GFX11-PAL-NEXT: global_store_b32 v[3:4], v6, off offset:3920 +; GFX11-PAL-NEXT: global_store_b32 v[16:17], v12, off offset:3920 +; GFX11-PAL-NEXT: global_store_b32 v[10:11], v13, off offset:3920 +; GFX11-PAL-NEXT: global_store_b32 v[18:19], v14, off offset:3920 +; GFX11-PAL-NEXT: global_store_b32 v[8:9], v15, off offset:3920 +; GFX11-PAL-NEXT: s_endpgm +; +; GFX11-PAL-GCNTRACKERS-LABEL: excess_soft_clause_reg_pressure: +; GFX11-PAL-GCNTRACKERS: ; %bb.0: ; %entry +; GFX11-PAL-GCNTRACKERS-NEXT: s_load_b256 s[16:23], s[4:5], 0x0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v11, 0 :: v_dual_and_b32 v34, 0x3ff, v0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v6, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v21, 0 :: v_dual_lshlrev_b32 v2, 2, v34 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v7, 0 :: v_dual_mov_b32 v8, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v10, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v17, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v19, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v18, 0 :: v_dual_mov_b32 v23, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v20, 0 :: v_dual_mov_b32 v25, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v22, 0 :: v_dual_mov_b32 v27, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v2, s0, s20, v2 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v3, null, s21, 0, s0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v29, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v4, vcc_lo, 0x188, v2 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v3, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v26, 0 :: v_dual_mov_b32 v3, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v28, 0 :: v_dual_mov_b32 v31, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v33, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v30, 0 :: v_dual_mov_b32 v13, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v32, 0 :: v_dual_mov_b32 v15, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v12, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v14, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 s33, s13 +; GFX11-PAL-GCNTRACKERS-NEXT: s_movk_i32 s34, 0x200 +; GFX11-PAL-GCNTRACKERS-NEXT: ; implicit-def: $vgpr50 : SGPR spill to VGPR lane +; GFX11-PAL-GCNTRACKERS-NEXT: .LBB4_1: ; %for.cond28.preheader +; GFX11-PAL-GCNTRACKERS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x3 +; GFX11-PAL-GCNTRACKERS-NEXT: global_load_b32 v37, v[4:5], off offset:-392 +; GFX11-PAL-GCNTRACKERS-NEXT: global_load_b32 v38, v[4:5], off offset:-196 +; GFX11-PAL-GCNTRACKERS-NEXT: global_load_b32 v36, v[4:5], off +; GFX11-PAL-GCNTRACKERS-NEXT: global_load_b32 v35, v[4:5], off offset:196 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x4 +; GFX11-PAL-GCNTRACKERS-NEXT: s_load_b512 s[84:99], s[16:17], 0x0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_load_b512 s[52:67], s[16:17], 0x1000 +; GFX11-PAL-GCNTRACKERS-NEXT: s_load_b512 s[36:51], s[16:17], 0x2000 +; GFX11-PAL-GCNTRACKERS-NEXT: s_load_b512 s[0:15], s[16:17], 0x3000 +; GFX11-PAL-GCNTRACKERS-NEXT: s_load_b512 s[68:83], s[16:17], 0x40 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v4, vcc_lo, 0x310, v4 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s0, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s1, 1 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s2, 2 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s3, 3 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s4, 4 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s5, 5 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s6, 6 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s7, 7 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s8, 8 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s9, 9 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s10, 10 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s11, 11 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s12, 12 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s13, 13 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s14, 14 +; GFX11-PAL-GCNTRACKERS-NEXT: v_writelane_b32 v50, s15, 15 +; GFX11-PAL-GCNTRACKERS-NEXT: s_load_b512 s[0:15], s[16:17], 0x1040 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(3) +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v32, s85, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v33, s84, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v30, s87, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v31, s86, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v3, s89, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v2, s88, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v28, s92, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v1, s90, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v26, s94, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v29, s91, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v24, s96, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v27, s93, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v22, s98, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v25, s95, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v20, s68, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v23, s97, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v18, s70, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v21, s99, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: s_load_b512 s[84:99], s[16:17], 0x2040 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v16, s72, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v19, s69, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v0, s74, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v17, s71, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v10, s75, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v11, s73, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v8, s77, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v9, s76, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v6, s79, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v7, s78, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v13, s81, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v12, s80, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v15, s83, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v14, s82, v37 +; GFX11-PAL-GCNTRACKERS-NEXT: s_load_b512 s[68:83], s[16:17], 0x3040 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(2) +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v32, s53, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v33, s52, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v30, s55, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v31, s54, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v3, s57, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v2, s56, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v28, s60, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v1, s58, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v26, s62, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v29, s59, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v24, s64, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v27, s61, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v22, s66, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v25, s63, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v20, s0, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v23, s65, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v18, s2, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v21, s67, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v16, s4, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v19, s1, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v0, s6, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v17, s3, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v10, s7, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v11, s5, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v8, s9, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v9, s8, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v6, s11, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v7, s10, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v13, s13, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v12, s12, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v15, s15, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v14, s14, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s0, v50, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v32, s37, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v16, s88, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v19, s85, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v0, s90, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v17, s87, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v10, s91, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v11, s89, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v8, s93, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v9, s92, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v6, s95, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v7, s94, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v13, s97, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v12, s96, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v15, s99, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v14, s98, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s1, v50, 1 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v33, s36, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s2, v50, 2 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v30, s39, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s3, v50, 3 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v31, s38, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s4, v50, 4 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v3, s41, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s5, v50, 5 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v2, s40, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s6, v50, 6 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v28, s44, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s7, v50, 7 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v26, s46, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s8, v50, 8 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v1, s42, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s9, v50, 9 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v24, s48, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s10, v50, 10 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v29, s43, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s11, v50, 11 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v22, s50, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s12, v50, 12 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v27, s45, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s13, v50, 13 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v20, s84, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v23, s49, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s14, v50, 14 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v25, s47, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_readlane_b32 s15, v50, 15 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v18, s86, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v21, s51, v36 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_u32 s16, s16, 0x4000 +; GFX11-PAL-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v32, s1, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v33, s0, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v30, s3, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v31, s2, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v3, s5, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v2, s4, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v28, s8, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v1, s6, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v26, s10, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v29, s7, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v24, s12, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v27, s9, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v22, s14, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v25, s11, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v20, s68, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v23, s13, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v18, s70, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v21, s15, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v16, s72, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v19, s69, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v0, s74, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v17, s71, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v10, s75, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v11, s73, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v8, s77, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v9, s76, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v6, s79, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v7, s78, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v13, s81, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v12, s80, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v15, s83, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: v_fmac_f32_e32 v14, s82, v35 +; GFX11-PAL-GCNTRACKERS-NEXT: s_addc_u32 s17, s17, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_add_i32 s34, s34, -1 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-PAL-GCNTRACKERS-NEXT: s_cmp_lg_u32 s34, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_cbranch_scc1 .LBB4_1 +; GFX11-PAL-GCNTRACKERS-NEXT: ; %bb.2: ; %for.cond.cleanup26 +; GFX11-PAL-GCNTRACKERS-NEXT: s_lshl_b32 s0, s33, 8 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add3_u32 v4, s22, s0, v34 +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 s0, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, 0 +; GFX11-PAL-GCNTRACKERS-NEXT: s_mov_b32 s1, s0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_dual_mov_b32 v36, s0 :: v_dual_mov_b32 v37, s1 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mul_hi_u32 v4, 0x5397829d, v4 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_lshrrev_b32_e32 v4, 4, v4 +; GFX11-PAL-GCNTRACKERS-NEXT: v_mul_lo_u32 v4, 0x31000, v4 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-PAL-GCNTRACKERS-NEXT: v_lshlrev_b64 v[34:35], 2, v[4:5] +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v34, vcc_lo, s18, v34 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v35, null, s19, v35, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x5 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[34:35], v33, off +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[34:35], v32, off offset:784 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b64 v[34:35], v[36:37], off offset:840 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[34:35], v31, off offset:1568 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[34:35], v30, off offset:2352 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[34:35], v2, off offset:3136 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v38, vcc_lo, 0x310, v34 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v39, null, 0, v35, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v40, vcc_lo, 0x620, v34 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v41, null, 0, v35, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v42, vcc_lo, 0x930, v34 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v43, null, 0, v35, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v44, vcc_lo, 0xc40, v34 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v45, null, 0, v35, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v30, vcc_lo, 0xf50, v34 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v31, null, 0, v35, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v32, vcc_lo, 0xf50, v38 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v33, null, 0, v39, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s0 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v46, vcc_lo, 0xf50, v40 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v47, null, 0, v41, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v48, vcc_lo, 0xf50, v42 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v43, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x3 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b64 v[34:35], v[3:4], off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b64 v[38:39], v[1:2], off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[40:41], v29, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[42:43], v28, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v1, vcc_lo, 0xf50, v44 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v2, null, 0, v45, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v3, vcc_lo, 0xf50, v30 +; GFX11-PAL-GCNTRACKERS-NEXT: s_clause 0x1 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b64 v[42:43], v[36:37], off offset:3976 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[44:45], v27, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v31, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[30:31], v26, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v26, vcc_lo, 0xf50, v32 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v27, null, 0, v33, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v28, vcc_lo, 0xf50, v46 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v29, null, 0, v47, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[46:47], v24, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v24, vcc_lo, 0xf50, v48 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[32:33], v25, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v25, null, 0, v49, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v30, vcc_lo, 0xf50, v1 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v31, null, 0, v2, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[1:2], v22, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v1, vcc_lo, 0xf50, v3 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v2, null, 0, v4, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[3:4], v21, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v3, vcc_lo, 0xf50, v26 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v27, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[26:27], v20, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v20, vcc_lo, 0xf50, v28 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v21, null, 0, v29, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v22, vcc_lo, 0xf50, v24 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[48:49], v23, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v23, null, 0, v25, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[24:25], v18, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v18, vcc_lo, 0xf50, v30 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[28:29], v19, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v19, null, 0, v31, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v24, vcc_lo, 0xf50, v1 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[30:31], v17, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v25, null, 0, v2, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[1:2], v16, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v1, vcc_lo, 0xf50, v3 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v2, null, 0, v4, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[3:4], v11, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v3, vcc_lo, 0xf50, v20 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v21, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v16, vcc_lo, 0xf50, v22 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v17, null, 0, v23, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[22:23], v10, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v10, vcc_lo, 0xf50, v18 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v19, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[18:19], v9, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v18, vcc_lo, 0xf50, v24 +; GFX11-PAL-GCNTRACKERS-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v19, null, 0, v25, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[24:25], v8, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_u32 v8, vcc_lo, 0xf50, v1 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[20:21], v0, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: v_add_co_ci_u32_e64 v9, null, 0, v2, vcc_lo +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[1:2], v7, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[3:4], v6, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[16:17], v12, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[10:11], v13, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[18:19], v14, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: global_store_b32 v[8:9], v15, off offset:3920 +; GFX11-PAL-GCNTRACKERS-NEXT: s_endpgm +; +; TONGA-LABEL: excess_soft_clause_reg_pressure: +; TONGA: ; %bb.0: ; %entry +; TONGA-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24 +; TONGA-NEXT: v_lshlrev_b32_e32 v1, 2, v0 +; TONGA-NEXT: s_waitcnt lgkmcnt(0) +; TONGA-NEXT: s_movk_i32 s7, 0x188 +; TONGA-NEXT: s_movk_i32 s9, 0x24c +; TONGA-NEXT: s_movk_i32 s10, 0x310 +; TONGA-NEXT: v_mov_b32_e32 v2, s5 +; TONGA-NEXT: v_add_u32_e32 v1, vcc, s4, v1 +; TONGA-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc +; TONGA-NEXT: s_movk_i32 s4, 0x200 +; TONGA-NEXT: s_movk_i32 s5, 0xc4 +; TONGA-NEXT: v_mov_b32_e32 v11, 0 +; TONGA-NEXT: v_mov_b32_e32 v13, 0 +; TONGA-NEXT: v_mov_b32_e32 v15, 0 +; TONGA-NEXT: v_mov_b32_e32 v18, 0 +; TONGA-NEXT: v_mov_b32_e32 v19, 0 +; TONGA-NEXT: v_mov_b32_e32 v20, 0 +; TONGA-NEXT: v_mov_b32_e32 v21, 0 +; TONGA-NEXT: v_mov_b32_e32 v22, 0 +; TONGA-NEXT: v_mov_b32_e32 v23, 0 +; TONGA-NEXT: v_mov_b32_e32 v24, 0 +; TONGA-NEXT: v_mov_b32_e32 v25, 0 +; TONGA-NEXT: v_mov_b32_e32 v26, 0 +; TONGA-NEXT: v_mov_b32_e32 v27, 0 +; TONGA-NEXT: v_mov_b32_e32 v28, 0 +; TONGA-NEXT: v_mov_b32_e32 v29, 0 +; TONGA-NEXT: v_mov_b32_e32 v30, 0 +; TONGA-NEXT: v_mov_b32_e32 v31, 0 +; TONGA-NEXT: v_mov_b32_e32 v32, 0 +; TONGA-NEXT: v_mov_b32_e32 v33, 0 +; TONGA-NEXT: v_mov_b32_e32 v34, 0 +; TONGA-NEXT: v_mov_b32_e32 v35, 0 +; TONGA-NEXT: v_mov_b32_e32 v36, 0 +; TONGA-NEXT: v_mov_b32_e32 v37, 0 +; TONGA-NEXT: v_mov_b32_e32 v38, 0 +; TONGA-NEXT: v_mov_b32_e32 v39, 0 +; TONGA-NEXT: v_mov_b32_e32 v40, 0 +; TONGA-NEXT: v_mov_b32_e32 v17, 0 +; TONGA-NEXT: v_mov_b32_e32 v16, 0 +; TONGA-NEXT: v_mov_b32_e32 v14, 0 +; TONGA-NEXT: v_mov_b32_e32 v12, 0 +; TONGA-NEXT: v_mov_b32_e32 v3, 0 +; TONGA-NEXT: v_mov_b32_e32 v4, 0 +; TONGA-NEXT: .LBB4_1: ; %for.cond28.preheader +; TONGA-NEXT: ; =>This Inner Loop Header: Depth=1 +; TONGA-NEXT: flat_load_dword v41, v[1:2] +; TONGA-NEXT: v_add_u32_e32 v5, vcc, s5, v1 +; TONGA-NEXT: v_addc_u32_e32 v6, vcc, 0, v2, vcc +; TONGA-NEXT: v_add_u32_e32 v7, vcc, s7, v1 +; TONGA-NEXT: v_addc_u32_e32 v8, vcc, 0, v2, vcc +; TONGA-NEXT: v_add_u32_e32 v9, vcc, s9, v1 +; TONGA-NEXT: v_addc_u32_e32 v10, vcc, 0, v2, vcc +; TONGA-NEXT: flat_load_dword v42, v[5:6] +; TONGA-NEXT: flat_load_dword v6, v[7:8] +; TONGA-NEXT: flat_load_dword v5, v[9:10] +; TONGA-NEXT: s_load_dwordx16 s[12:27], s[0:1], 0x0 +; TONGA-NEXT: s_load_dwordx16 s[36:51], s[0:1], 0x40 +; TONGA-NEXT: s_load_dwordx16 s[52:67], s[0:1], 0x1000 +; TONGA-NEXT: s_load_dwordx16 s[68:83], s[0:1], 0x1040 +; TONGA-NEXT: v_add_u32_e32 v1, vcc, s10, v1 +; TONGA-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc +; TONGA-NEXT: s_waitcnt vmcnt(3) lgkmcnt(0) +; TONGA-NEXT: v_mul_f32_e32 v7, s17, v41 +; TONGA-NEXT: v_mul_f32_e32 v8, s18, v41 +; TONGA-NEXT: v_mul_f32_e32 v9, s12, v41 +; TONGA-NEXT: v_add_f32_e32 v43, v7, v4 +; TONGA-NEXT: v_mul_f32_e32 v4, s14, v41 +; TONGA-NEXT: v_mul_f32_e32 v7, s15, v41 +; TONGA-NEXT: v_add_f32_e32 v44, v8, v3 +; TONGA-NEXT: v_mul_f32_e32 v3, s16, v41 +; TONGA-NEXT: v_mul_f32_e32 v10, s13, v41 +; TONGA-NEXT: v_mul_f32_e32 v8, s19, v41 +; TONGA-NEXT: v_add_f32_e32 v48, v9, v40 +; TONGA-NEXT: v_mul_f32_e32 v9, s20, v41 +; TONGA-NEXT: v_add_f32_e32 v50, v4, v38 +; TONGA-NEXT: v_mul_f32_e32 v4, s22, v41 +; TONGA-NEXT: v_add_f32_e32 v51, v7, v37 +; TONGA-NEXT: v_mul_f32_e32 v7, s23, v41 +; TONGA-NEXT: v_add_f32_e32 v52, v3, v36 +; TONGA-NEXT: v_mul_f32_e32 v3, s24, v41 +; TONGA-NEXT: v_add_f32_e32 v49, v10, v39 +; TONGA-NEXT: v_mul_f32_e32 v10, s21, v41 +; TONGA-NEXT: v_add_f32_e32 v35, v8, v35 +; TONGA-NEXT: v_mul_f32_e32 v8, s25, v41 +; TONGA-NEXT: v_add_f32_e32 v34, v9, v34 +; TONGA-NEXT: v_mul_f32_e32 v9, s26, v41 +; TONGA-NEXT: v_add_f32_e32 v32, v4, v32 +; TONGA-NEXT: v_mul_f32_e32 v4, s36, v41 +; TONGA-NEXT: v_add_f32_e32 v31, v7, v31 +; TONGA-NEXT: v_mul_f32_e32 v7, s37, v41 +; TONGA-NEXT: v_add_f32_e32 v30, v3, v30 +; TONGA-NEXT: v_mul_f32_e32 v3, s38, v41 +; TONGA-NEXT: v_add_f32_e32 v33, v10, v33 +; TONGA-NEXT: v_mul_f32_e32 v10, s27, v41 +; TONGA-NEXT: s_load_dwordx16 s[12:27], s[0:1], 0x2000 +; TONGA-NEXT: v_add_f32_e32 v29, v8, v29 +; TONGA-NEXT: v_mul_f32_e32 v8, s39, v41 +; TONGA-NEXT: v_add_f32_e32 v28, v9, v28 +; TONGA-NEXT: v_mul_f32_e32 v9, s40, v41 +; TONGA-NEXT: v_add_f32_e32 v37, v4, v26 +; TONGA-NEXT: v_mul_f32_e32 v4, s42, v41 +; TONGA-NEXT: v_add_f32_e32 v38, v7, v25 +; TONGA-NEXT: v_mul_f32_e32 v7, s43, v41 +; TONGA-NEXT: v_add_f32_e32 v39, v3, v24 +; TONGA-NEXT: v_mul_f32_e32 v3, s44, v41 +; TONGA-NEXT: v_add_f32_e32 v36, v10, v27 +; TONGA-NEXT: v_mul_f32_e32 v10, s41, v41 +; TONGA-NEXT: v_add_f32_e32 v40, v8, v23 +; TONGA-NEXT: v_mul_f32_e32 v8, s45, v41 +; TONGA-NEXT: v_add_f32_e32 v22, v9, v22 +; TONGA-NEXT: v_mul_f32_e32 v9, s46, v41 +; TONGA-NEXT: v_add_f32_e32 v46, v4, v20 +; TONGA-NEXT: v_mul_f32_e32 v4, s48, v41 +; TONGA-NEXT: v_add_f32_e32 v47, v7, v19 +; TONGA-NEXT: v_mul_f32_e32 v7, s49, v41 +; TONGA-NEXT: v_add_f32_e32 v3, v3, v18 +; TONGA-NEXT: v_mul_f32_e32 v18, s50, v41 +; TONGA-NEXT: v_mul_f32_e32 v19, s51, v41 +; TONGA-NEXT: s_waitcnt vmcnt(2) +; TONGA-NEXT: v_mul_f32_e32 v23, s61, v42 +; TONGA-NEXT: v_add_f32_e32 v45, v10, v21 +; TONGA-NEXT: v_mul_f32_e32 v10, s47, v41 +; TONGA-NEXT: s_load_dwordx16 s[36:51], s[0:1], 0x2040 +; TONGA-NEXT: v_add_f32_e32 v21, v8, v15 +; TONGA-NEXT: v_add_f32_e32 v13, v9, v13 +; TONGA-NEXT: v_add_f32_e32 v9, v4, v17 +; TONGA-NEXT: v_add_f32_e32 v8, v7, v16 +; TONGA-NEXT: v_add_f32_e32 v7, v18, v14 +; TONGA-NEXT: v_add_f32_e32 v4, v19, v12 +; TONGA-NEXT: v_mul_f32_e32 v17, s57, v42 +; TONGA-NEXT: v_mul_f32_e32 v18, s58, v42 +; TONGA-NEXT: v_mul_f32_e32 v19, s59, v42 +; TONGA-NEXT: v_mul_f32_e32 v20, s60, v42 +; TONGA-NEXT: v_mul_f32_e32 v24, s62, v42 +; TONGA-NEXT: v_mul_f32_e32 v26, s64, v42 +; TONGA-NEXT: v_add_f32_e32 v23, v23, v33 +; TONGA-NEXT: v_mul_f32_e32 v33, s68, v42 +; TONGA-NEXT: v_add_f32_e32 v10, v10, v11 +; TONGA-NEXT: v_mul_f32_e32 v11, s52, v42 +; TONGA-NEXT: v_mul_f32_e32 v12, s53, v42 +; TONGA-NEXT: v_mul_f32_e32 v14, s54, v42 +; TONGA-NEXT: v_mul_f32_e32 v15, s55, v42 +; TONGA-NEXT: v_mul_f32_e32 v16, s56, v42 +; TONGA-NEXT: v_mul_f32_e32 v25, s63, v42 +; TONGA-NEXT: v_add_f32_e32 v17, v17, v43 +; TONGA-NEXT: v_add_f32_e32 v18, v18, v44 +; TONGA-NEXT: v_mul_f32_e32 v27, s65, v42 +; TONGA-NEXT: v_add_f32_e32 v19, v19, v35 +; TONGA-NEXT: v_mul_f32_e32 v35, s66, v42 +; TONGA-NEXT: v_add_f32_e32 v20, v20, v34 +; TONGA-NEXT: v_mul_f32_e32 v34, s67, v42 +; TONGA-NEXT: s_load_dwordx16 s[52:67], s[0:1], 0x3000 +; TONGA-NEXT: v_add_f32_e32 v24, v24, v32 +; TONGA-NEXT: v_mul_f32_e32 v32, s69, v42 +; TONGA-NEXT: v_mul_f32_e32 v41, s70, v42 +; TONGA-NEXT: v_add_f32_e32 v26, v26, v30 +; TONGA-NEXT: v_mul_f32_e32 v43, s71, v42 +; TONGA-NEXT: v_mul_f32_e32 v44, s72, v42 +; TONGA-NEXT: v_add_f32_e32 v30, v33, v37 +; TONGA-NEXT: v_mul_f32_e32 v37, s75, v42 +; TONGA-NEXT: v_add_f32_e32 v25, v25, v31 +; TONGA-NEXT: v_add_f32_e32 v27, v27, v29 +; TONGA-NEXT: v_add_f32_e32 v29, v34, v36 +; TONGA-NEXT: v_add_f32_e32 v31, v32, v38 +; TONGA-NEXT: v_mul_f32_e32 v38, s76, v42 +; TONGA-NEXT: v_add_f32_e32 v32, v41, v39 +; TONGA-NEXT: v_mul_f32_e32 v39, s77, v42 +; TONGA-NEXT: v_add_f32_e32 v33, v43, v40 +; TONGA-NEXT: v_mul_f32_e32 v40, s78, v42 +; TONGA-NEXT: v_add_f32_e32 v34, v44, v22 +; TONGA-NEXT: v_mul_f32_e32 v41, s79, v42 +; TONGA-NEXT: v_mul_f32_e32 v43, s80, v42 +; TONGA-NEXT: v_mul_f32_e32 v44, s81, v42 +; TONGA-NEXT: v_add_f32_e32 v22, v37, v47 +; TONGA-NEXT: v_mul_f32_e32 v37, s82, v42 +; TONGA-NEXT: v_add_f32_e32 v11, v11, v48 +; TONGA-NEXT: v_add_f32_e32 v12, v12, v49 +; TONGA-NEXT: v_add_f32_e32 v14, v14, v50 +; TONGA-NEXT: v_add_f32_e32 v15, v15, v51 +; TONGA-NEXT: v_add_f32_e32 v16, v16, v52 +; TONGA-NEXT: v_add_f32_e32 v28, v35, v28 +; TONGA-NEXT: v_mul_f32_e32 v35, s73, v42 +; TONGA-NEXT: v_mul_f32_e32 v36, s74, v42 +; TONGA-NEXT: v_mul_f32_e32 v42, s83, v42 +; TONGA-NEXT: s_load_dwordx16 s[68:83], s[0:1], 0x3040 +; TONGA-NEXT: v_add_f32_e32 v3, v38, v3 +; TONGA-NEXT: s_waitcnt vmcnt(1) lgkmcnt(0) +; TONGA-NEXT: v_mul_f32_e32 v38, s12, v6 +; TONGA-NEXT: v_add_f32_e32 v21, v39, v21 +; TONGA-NEXT: v_mul_f32_e32 v39, s13, v6 +; TONGA-NEXT: v_add_f32_e32 v13, v40, v13 +; TONGA-NEXT: v_mul_f32_e32 v40, s14, v6 +; TONGA-NEXT: v_add_f32_e32 v10, v41, v10 +; TONGA-NEXT: v_mul_f32_e32 v41, s15, v6 +; TONGA-NEXT: v_add_f32_e32 v9, v43, v9 +; TONGA-NEXT: v_mul_f32_e32 v43, s16, v6 +; TONGA-NEXT: v_add_f32_e32 v8, v44, v8 +; TONGA-NEXT: v_mul_f32_e32 v44, s17, v6 +; TONGA-NEXT: v_add_f32_e32 v7, v37, v7 +; TONGA-NEXT: v_mul_f32_e32 v37, s18, v6 +; TONGA-NEXT: v_add_f32_e32 v35, v35, v45 +; TONGA-NEXT: v_add_f32_e32 v4, v42, v4 +; TONGA-NEXT: v_mul_f32_e32 v42, s19, v6 +; TONGA-NEXT: v_add_f32_e32 v11, v38, v11 +; TONGA-NEXT: v_mul_f32_e32 v38, s20, v6 +; TONGA-NEXT: v_add_f32_e32 v12, v39, v12 +; TONGA-NEXT: v_mul_f32_e32 v39, s21, v6 +; TONGA-NEXT: v_add_f32_e32 v14, v40, v14 +; TONGA-NEXT: v_mul_f32_e32 v40, s22, v6 +; TONGA-NEXT: v_add_f32_e32 v15, v41, v15 +; TONGA-NEXT: v_mul_f32_e32 v41, s23, v6 +; TONGA-NEXT: v_add_f32_e32 v16, v43, v16 +; TONGA-NEXT: v_mul_f32_e32 v43, s24, v6 +; TONGA-NEXT: v_add_f32_e32 v44, v44, v17 +; TONGA-NEXT: v_mul_f32_e32 v17, s25, v6 +; TONGA-NEXT: v_add_f32_e32 v45, v37, v18 +; TONGA-NEXT: v_mul_f32_e32 v18, s26, v6 +; TONGA-NEXT: v_mul_f32_e32 v37, s27, v6 +; TONGA-NEXT: v_add_f32_e32 v36, v36, v46 +; TONGA-NEXT: v_add_f32_e32 v19, v42, v19 +; TONGA-NEXT: v_add_f32_e32 v20, v38, v20 +; TONGA-NEXT: v_mul_f32_e32 v38, s36, v6 +; TONGA-NEXT: v_add_f32_e32 v23, v39, v23 +; TONGA-NEXT: v_mul_f32_e32 v39, s37, v6 +; TONGA-NEXT: v_add_f32_e32 v24, v40, v24 +; TONGA-NEXT: v_mul_f32_e32 v40, s38, v6 +; TONGA-NEXT: v_add_f32_e32 v25, v41, v25 +; TONGA-NEXT: v_mul_f32_e32 v41, s39, v6 +; TONGA-NEXT: v_add_f32_e32 v26, v43, v26 +; TONGA-NEXT: v_mul_f32_e32 v42, s40, v6 +; TONGA-NEXT: v_add_f32_e32 v17, v17, v27 +; TONGA-NEXT: v_mul_f32_e32 v27, s41, v6 +; TONGA-NEXT: v_add_f32_e32 v18, v18, v28 +; TONGA-NEXT: v_mul_f32_e32 v28, s42, v6 +; TONGA-NEXT: v_add_f32_e32 v43, v37, v29 +; TONGA-NEXT: v_mul_f32_e32 v29, s43, v6 +; TONGA-NEXT: v_add_f32_e32 v46, v38, v30 +; TONGA-NEXT: v_mul_f32_e32 v30, s44, v6 +; TONGA-NEXT: v_add_f32_e32 v47, v39, v31 +; TONGA-NEXT: v_mul_f32_e32 v31, s45, v6 +; TONGA-NEXT: v_add_f32_e32 v48, v40, v32 +; TONGA-NEXT: v_mul_f32_e32 v32, s46, v6 +; TONGA-NEXT: v_add_f32_e32 v41, v41, v33 +; TONGA-NEXT: v_mul_f32_e32 v33, s47, v6 +; TONGA-NEXT: v_add_f32_e32 v42, v42, v34 +; TONGA-NEXT: v_mul_f32_e32 v34, s48, v6 +; TONGA-NEXT: v_add_f32_e32 v49, v27, v35 +; TONGA-NEXT: v_mul_f32_e32 v27, s49, v6 +; TONGA-NEXT: v_add_f32_e32 v50, v28, v36 +; TONGA-NEXT: v_mul_f32_e32 v28, s50, v6 +; TONGA-NEXT: v_mul_f32_e32 v6, s51, v6 +; TONGA-NEXT: v_add_f32_e32 v51, v29, v22 +; TONGA-NEXT: s_waitcnt vmcnt(0) +; TONGA-NEXT: v_mul_f32_e32 v22, s52, v5 +; TONGA-NEXT: v_mul_f32_e32 v29, s53, v5 +; TONGA-NEXT: v_add_f32_e32 v3, v30, v3 +; TONGA-NEXT: v_add_f32_e32 v52, v31, v21 +; TONGA-NEXT: v_mul_f32_e32 v21, s54, v5 +; TONGA-NEXT: v_add_f32_e32 v13, v32, v13 +; TONGA-NEXT: v_mul_f32_e32 v30, s55, v5 +; TONGA-NEXT: v_mul_f32_e32 v31, s56, v5 +; TONGA-NEXT: v_mul_f32_e32 v32, s59, v5 +; TONGA-NEXT: v_add_f32_e32 v8, v27, v8 +; TONGA-NEXT: v_mul_f32_e32 v27, s60, v5 +; TONGA-NEXT: v_add_f32_e32 v4, v6, v4 +; TONGA-NEXT: v_mul_f32_e32 v6, s62, v5 +; TONGA-NEXT: v_add_f32_e32 v40, v22, v11 +; TONGA-NEXT: v_mul_f32_e32 v11, s63, v5 +; TONGA-NEXT: v_add_f32_e32 v39, v29, v12 +; TONGA-NEXT: v_mul_f32_e32 v12, s64, v5 +; TONGA-NEXT: s_add_u32 s0, s0, 0x4000 +; TONGA-NEXT: v_add_f32_e32 v9, v34, v9 +; TONGA-NEXT: v_add_f32_e32 v7, v28, v7 +; TONGA-NEXT: v_mul_f32_e32 v28, s61, v5 +; TONGA-NEXT: v_add_f32_e32 v38, v21, v14 +; TONGA-NEXT: v_mul_f32_e32 v14, s65, v5 +; TONGA-NEXT: v_add_f32_e32 v37, v30, v15 +; TONGA-NEXT: v_mul_f32_e32 v15, s66, v5 +; TONGA-NEXT: v_add_f32_e32 v36, v31, v16 +; TONGA-NEXT: v_mul_f32_e32 v16, s67, v5 +; TONGA-NEXT: v_add_f32_e32 v35, v32, v19 +; TONGA-NEXT: v_mul_f32_e32 v19, s68, v5 +; TONGA-NEXT: v_add_f32_e32 v34, v27, v20 +; TONGA-NEXT: v_mul_f32_e32 v20, s69, v5 +; TONGA-NEXT: v_mul_f32_e32 v21, s70, v5 +; TONGA-NEXT: v_add_f32_e32 v32, v6, v24 +; TONGA-NEXT: v_mul_f32_e32 v6, s71, v5 +; TONGA-NEXT: v_add_f32_e32 v31, v11, v25 +; TONGA-NEXT: v_mul_f32_e32 v11, s72, v5 +; TONGA-NEXT: v_add_f32_e32 v30, v12, v26 +; TONGA-NEXT: v_mul_f32_e32 v12, s73, v5 +; TONGA-NEXT: s_addc_u32 s1, s1, 0 +; TONGA-NEXT: s_add_i32 s4, s4, -1 +; TONGA-NEXT: v_add_f32_e32 v10, v33, v10 +; TONGA-NEXT: v_add_f32_e32 v33, v28, v23 +; TONGA-NEXT: v_add_f32_e32 v29, v14, v17 +; TONGA-NEXT: v_mul_f32_e32 v14, s74, v5 +; TONGA-NEXT: v_add_f32_e32 v28, v15, v18 +; TONGA-NEXT: v_mul_f32_e32 v15, s75, v5 +; TONGA-NEXT: v_add_f32_e32 v27, v16, v43 +; TONGA-NEXT: v_mul_f32_e32 v16, s76, v5 +; TONGA-NEXT: v_add_f32_e32 v26, v19, v46 +; TONGA-NEXT: v_mul_f32_e32 v17, s77, v5 +; TONGA-NEXT: v_add_f32_e32 v25, v20, v47 +; TONGA-NEXT: v_mul_f32_e32 v43, s78, v5 +; TONGA-NEXT: v_add_f32_e32 v24, v21, v48 +; TONGA-NEXT: v_mul_f32_e32 v46, s79, v5 +; TONGA-NEXT: v_add_f32_e32 v23, v6, v41 +; TONGA-NEXT: v_mul_f32_e32 v6, s80, v5 +; TONGA-NEXT: v_add_f32_e32 v22, v11, v42 +; TONGA-NEXT: v_mul_f32_e32 v41, s81, v5 +; TONGA-NEXT: v_add_f32_e32 v21, v12, v49 +; TONGA-NEXT: v_mul_f32_e32 v12, s82, v5 +; TONGA-NEXT: v_mul_f32_e32 v42, s83, v5 +; TONGA-NEXT: v_mul_f32_e32 v47, s57, v5 +; TONGA-NEXT: v_mul_f32_e32 v5, s58, v5 +; TONGA-NEXT: s_cmp_lg_u32 s4, 0 +; TONGA-NEXT: v_add_f32_e32 v20, v14, v50 +; TONGA-NEXT: v_add_f32_e32 v19, v15, v51 +; TONGA-NEXT: v_add_f32_e32 v18, v16, v3 +; TONGA-NEXT: v_add_f32_e32 v15, v17, v52 +; TONGA-NEXT: v_add_f32_e32 v13, v43, v13 +; TONGA-NEXT: v_add_f32_e32 v11, v46, v10 +; TONGA-NEXT: v_add_f32_e32 v17, v6, v9 +; TONGA-NEXT: v_add_f32_e32 v16, v41, v8 +; TONGA-NEXT: v_add_f32_e32 v14, v12, v7 +; TONGA-NEXT: v_add_f32_e32 v12, v42, v4 +; TONGA-NEXT: v_add_f32_e32 v4, v47, v44 +; TONGA-NEXT: v_add_f32_e32 v3, v5, v45 +; TONGA-NEXT: s_cbranch_scc1 .LBB4_1 +; TONGA-NEXT: ; %bb.2: ; %for.cond.cleanup26 +; TONGA-NEXT: s_lshl_b32 s0, s8, 8 +; TONGA-NEXT: s_add_i32 s0, s6, s0 +; TONGA-NEXT: v_add_u32_e32 v0, vcc, s0, v0 +; TONGA-NEXT: s_mov_b32 s0, 0x5397829d +; TONGA-NEXT: v_mul_hi_u32 v0, v0, s0 +; TONGA-NEXT: s_mov_b32 s0, 0x31000 +; TONGA-NEXT: v_mov_b32_e32 v1, 0 +; TONGA-NEXT: v_mov_b32_e32 v2, s3 +; TONGA-NEXT: v_lshrrev_b32_e32 v0, 4, v0 +; TONGA-NEXT: v_mul_lo_u32 v0, v0, s0 +; TONGA-NEXT: s_movk_i32 s0, 0x310 +; TONGA-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1] +; TONGA-NEXT: v_add_u32_e32 v0, vcc, s2, v0 +; TONGA-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc +; TONGA-NEXT: v_add_u32_e32 v5, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x348 +; TONGA-NEXT: flat_store_dword v[5:6], v39 +; TONGA-NEXT: v_add_u32_e32 v5, vcc, s0, v0 +; TONGA-NEXT: s_mov_b32 s0, 0 +; TONGA-NEXT: s_mov_b32 s1, s0 +; TONGA-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc +; TONGA-NEXT: v_mov_b32_e32 v8, s1 +; TONGA-NEXT: v_mov_b32_e32 v7, s0 +; TONGA-NEXT: s_movk_i32 s1, 0x620 +; TONGA-NEXT: flat_store_dwordx2 v[5:6], v[7:8] +; TONGA-NEXT: v_add_u32_e32 v5, vcc, s1, v0 +; TONGA-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s1, 0x930 +; TONGA-NEXT: flat_store_dword v[5:6], v38 +; TONGA-NEXT: v_add_u32_e32 v5, vcc, s1, v0 +; TONGA-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s1, 0xc40 +; TONGA-NEXT: flat_store_dword v[5:6], v37 +; TONGA-NEXT: v_add_u32_e32 v5, vcc, s1, v0 +; TONGA-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s1, 0xf50 +; TONGA-NEXT: v_add_u32_e32 v9, vcc, s1, v0 +; TONGA-NEXT: flat_store_dword v[5:6], v36 +; TONGA-NEXT: v_addc_u32_e32 v10, vcc, 0, v1, vcc +; TONGA-NEXT: v_mov_b32_e32 v5, s0 +; TONGA-NEXT: s_movk_i32 s1, 0x1260 +; TONGA-NEXT: flat_store_dwordx2 v[9:10], v[4:5] +; TONGA-NEXT: v_add_u32_e32 v5, vcc, s1, v0 +; TONGA-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc +; TONGA-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-NEXT: s_movk_i32 s0, 0x1570 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: flat_store_dwordx2 v[5:6], v[3:4] +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x1880 +; TONGA-NEXT: flat_store_dword v[2:3], v35 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x18b8 +; TONGA-NEXT: flat_store_dword v[2:3], v34 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x1b90 +; TONGA-NEXT: flat_store_dwordx2 v[2:3], v[7:8] +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x1ea0 +; TONGA-NEXT: flat_store_dword v[2:3], v33 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x21b0 +; TONGA-NEXT: flat_store_dword v[0:1], v40 +; TONGA-NEXT: flat_store_dword v[2:3], v32 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x24c0 +; TONGA-NEXT: flat_store_dword v[2:3], v31 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x27d0 +; TONGA-NEXT: flat_store_dword v[2:3], v30 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x2ae0 +; TONGA-NEXT: flat_store_dword v[2:3], v29 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x2df0 +; TONGA-NEXT: flat_store_dword v[2:3], v28 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x3100 +; TONGA-NEXT: flat_store_dword v[2:3], v27 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x3410 +; TONGA-NEXT: flat_store_dword v[2:3], v26 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x3720 +; TONGA-NEXT: flat_store_dword v[2:3], v25 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x3a30 +; TONGA-NEXT: flat_store_dword v[2:3], v24 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x3d40 +; TONGA-NEXT: flat_store_dword v[2:3], v23 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x4050 +; TONGA-NEXT: flat_store_dword v[2:3], v22 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x4360 +; TONGA-NEXT: flat_store_dword v[2:3], v21 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x4670 +; TONGA-NEXT: flat_store_dword v[2:3], v20 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x4980 +; TONGA-NEXT: flat_store_dword v[2:3], v19 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x4c90 +; TONGA-NEXT: flat_store_dword v[2:3], v18 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x4fa0 +; TONGA-NEXT: flat_store_dword v[2:3], v15 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x52b0 +; TONGA-NEXT: flat_store_dword v[2:3], v13 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x55c0 +; TONGA-NEXT: flat_store_dword v[2:3], v11 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: s_movk_i32 s0, 0x58d0 +; TONGA-NEXT: flat_store_dword v[2:3], v17 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: flat_store_dword v[2:3], v16 +; TONGA-NEXT: v_add_u32_e32 v2, vcc, 0x5be0, v0 +; TONGA-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-NEXT: v_add_u32_e32 v0, vcc, 0x5ef0, v0 +; TONGA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; TONGA-NEXT: flat_store_dword v[2:3], v14 +; TONGA-NEXT: flat_store_dword v[0:1], v12 +; TONGA-NEXT: s_endpgm +; +; TONGA-GCNTRACKERS-LABEL: excess_soft_clause_reg_pressure: +; TONGA-GCNTRACKERS: ; %bb.0: ; %entry +; TONGA-GCNTRACKERS-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24 +; TONGA-GCNTRACKERS-NEXT: ; implicit-def: $vgpr47 : SGPR spill to VGPR lane +; TONGA-GCNTRACKERS-NEXT: v_lshlrev_b32_e32 v1, 2, v0 +; TONGA-GCNTRACKERS-NEXT: s_mov_b32 s33, s8 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, 0 +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s34, 0x200 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s5 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s0, 0 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s1, 1 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s2, 2 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s3, 3 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s4, 4 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s5, 5 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v1, vcc, s4, v1 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s6, 6 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s35, 0xc4 +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s84, 0x188 +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s85, 0x24c +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s86, 0x310 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v11, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v13, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v16, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v18, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v19, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v20, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v21, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v22, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v23, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v24, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v25, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v26, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v27, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v28, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v29, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v30, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v31, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v32, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v33, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v34, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v35, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v36, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v37, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v38, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v39, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v40, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v17, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v15, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v14, 0 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v12, 0 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s7, 7 +; TONGA-GCNTRACKERS-NEXT: .LBB4_1: ; %for.cond28.preheader +; TONGA-GCNTRACKERS-NEXT: ; =>This Inner Loop Header: Depth=1 +; TONGA-GCNTRACKERS-NEXT: flat_load_dword v41, v[1:2] +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v5, vcc, s35, v1 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v6, vcc, 0, v2, vcc +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v7, vcc, s84, v1 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v8, vcc, 0, v2, vcc +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v9, vcc, s85, v1 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v10, vcc, 0, v2, vcc +; TONGA-GCNTRACKERS-NEXT: flat_load_dword v42, v[5:6] +; TONGA-GCNTRACKERS-NEXT: flat_load_dword v6, v[7:8] +; TONGA-GCNTRACKERS-NEXT: flat_load_dword v5, v[9:10] +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s68, v47, 0 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s69, v47, 1 +; TONGA-GCNTRACKERS-NEXT: s_load_dwordx16 s[12:27], s[68:69], 0x0 +; TONGA-GCNTRACKERS-NEXT: s_load_dwordx16 s[36:51], s[68:69], 0x40 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s70, v47, 2 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s71, v47, 3 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s72, v47, 4 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s73, v47, 5 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s74, v47, 6 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s75, v47, 7 +; TONGA-GCNTRACKERS-NEXT: s_load_dwordx16 s[52:67], s[68:69], 0x2040 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v1, vcc, s86, v1 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(3) lgkmcnt(0) +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v7, s12, v41 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v8, s13, v41 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v9, s14, v41 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v10, s15, v41 +; TONGA-GCNTRACKERS-NEXT: s_load_dwordx16 s[0:15], s[68:69], 0x2000 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v7, v7, v40 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s17, v41 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s16, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v8, v8, v39 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s0, 8 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s1, 9 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s2, 10 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s3, 11 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s4, 12 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s5, 13 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s6, 14 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s7, 15 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s8, 16 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s9, 17 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s10, 18 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s11, 19 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s12, 20 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s13, 21 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s14, 22 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s15, 23 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v39, s18, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v9, v9, v38 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v38, s19, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v10, v10, v37 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s20, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v4, v40, v4 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s22, v41 +; TONGA-GCNTRACKERS-NEXT: s_load_dwordx16 s[0:15], s[68:69], 0x1000 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v36, v43, v36 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s21, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v3, v39, v3 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v39, s23, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v35, v38, v35 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v38, s24, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v34, v37, v34 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s25, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v32, v40, v32 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s27, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v33, v43, v33 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s26, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v31, v39, v31 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v39, s36, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v30, v38, v30 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v38, s37, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v29, v37, v29 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s38, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v27, v40, v27 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s40, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v28, v43, v28 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s39, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v26, v39, v26 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v39, s41, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v25, v38, v25 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v38, s42, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v24, v37, v24 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s43, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v22, v40, v22 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s45, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v23, v43, v23 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s44, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v21, v39, v21 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v39, s46, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v20, v38, v20 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v38, s47, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v19, v37, v19 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s48, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v16, v40, v16 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s50, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v18, v43, v18 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s49, v41 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v41, s51, v41 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v13, v39, v13 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v11, v38, v11 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v17, v37, v17 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v14, v40, v14 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(2) lgkmcnt(0) +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s0, v42 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v38, s1, v42 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v39, s2, v42 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s3, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v15, v43, v15 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v12, v41, v12 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v41, s4, v42 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s5, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v7, v37, v7 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s6, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v8, v38, v8 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v38, s7, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v9, v39, v9 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v39, s8, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v10, v40, v10 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s9, v42 +; TONGA-GCNTRACKERS-NEXT: s_load_dwordx16 s[36:51], s[68:69], 0x3000 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v36, v41, v36 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v41, s10, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v4, v43, v4 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s11, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v3, v37, v3 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s12, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v35, v38, v35 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v38, s13, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v34, v39, v34 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v39, s14, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v33, v40, v33 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s15, v42 +; TONGA-GCNTRACKERS-NEXT: s_load_dwordx16 s[0:15], s[68:69], 0x1040 +; TONGA-GCNTRACKERS-NEXT: s_load_dwordx16 s[16:31], s[68:69], 0x3040 +; TONGA-GCNTRACKERS-NEXT: s_add_u32 s68, s68, 0x4000 +; TONGA-GCNTRACKERS-NEXT: s_addc_u32 s69, s69, 0 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s68, 0 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s69, 1 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s70, 2 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s71, 3 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s72, 4 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s73, 5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v32, v41, v32 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v41, s0, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v31, v43, v31 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s1, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v30, v37, v30 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s2, v42 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s74, 6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v29, v38, v29 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v38, s3, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v28, v39, v28 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v39, s4, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v27, v40, v27 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s5, v42 +; TONGA-GCNTRACKERS-NEXT: v_writelane_b32 v47, s75, 7 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v26, v41, v26 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v41, s6, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v25, v43, v25 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s7, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v24, v37, v24 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s8, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v23, v38, v23 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v38, s9, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v22, v39, v22 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v39, s10, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v21, v40, v21 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s11, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v20, v41, v20 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v41, s12, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v19, v43, v19 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s13, v42 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v18, v37, v18 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s14, v42 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v42, s15, v42 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s0, v47, 8 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s1, v47, 9 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s2, v47, 10 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s3, v47, 11 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s4, v47, 12 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s5, v47, 13 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s6, v47, 14 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v16, v38, v16 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s7, v47, 15 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s8, v47, 16 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s9, v47, 17 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s10, v47, 18 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s11, v47, 19 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s12, v47, 20 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s13, v47, 21 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(1) +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v38, s0, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v13, v39, v13 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v39, s1, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v11, v40, v11 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s2, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v17, v41, v17 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v41, s3, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v15, v43, v15 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s4, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v14, v37, v14 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s5, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v12, v42, v12 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v42, s6, v6 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s14, v47, 22 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s15, v47, 23 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v7, v38, v7 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v38, s7, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v8, v39, v8 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v39, s8, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v9, v40, v9 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s9, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v10, v41, v10 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v41, s10, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v36, v43, v36 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s11, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v4, v37, v4 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s12, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v3, v42, v3 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v42, s13, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v35, v38, v35 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v38, s14, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v34, v39, v34 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v39, s15, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v33, v40, v33 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s52, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v32, v41, v32 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v41, s53, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v31, v43, v31 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s54, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v30, v37, v30 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s55, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v29, v42, v29 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v42, s56, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v28, v38, v28 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v38, s57, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v27, v39, v27 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v39, s58, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v26, v40, v26 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s59, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v25, v41, v25 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v41, s60, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v24, v43, v24 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s61, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v23, v37, v23 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s62, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v22, v42, v22 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v42, s63, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v21, v38, v21 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v38, s64, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v20, v39, v20 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v39, s65, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v19, v40, v19 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v40, s66, v6 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v6, s67, v6 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v18, v41, v18 +; TONGA-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v41, s36, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v16, v43, v16 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s37, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v13, v37, v13 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v37, s38, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v11, v42, v11 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v42, s39, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v17, v38, v17 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v15, v39, v15 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v14, v40, v14 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v6, v6, v12 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v12, s43, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v40, v41, v7 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v7, s44, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v39, v43, v8 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v8, s45, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v38, v37, v9 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v9, s46, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v37, v42, v10 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v10, s47, v5 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v41, s48, v5 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v42, s49, v5 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s50, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v35, v12, v35 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v12, s51, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v34, v7, v34 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v7, s16, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v33, v8, v33 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v8, s17, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v32, v9, v32 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v9, s18, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v31, v10, v31 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v10, s19, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v30, v41, v30 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v41, s20, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v29, v42, v29 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v42, s21, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v28, v43, v28 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s22, v5 +; TONGA-GCNTRACKERS-NEXT: s_add_i32 s34, s34, -1 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v44, s40, v5 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v45, s41, v5 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v46, s42, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v27, v12, v27 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v12, s23, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v26, v7, v26 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v7, s24, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v25, v8, v25 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v8, s25, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v24, v9, v24 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v9, s26, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v23, v10, v23 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v10, s27, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v22, v41, v22 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v41, s28, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v21, v42, v21 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v42, s29, v5 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v20, v43, v20 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v43, s30, v5 +; TONGA-GCNTRACKERS-NEXT: v_mul_f32_e32 v5, s31, v5 +; TONGA-GCNTRACKERS-NEXT: s_cmp_lg_u32 s34, 0 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v36, v44, v36 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v4, v45, v4 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v3, v46, v3 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v19, v12, v19 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v18, v7, v18 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v16, v8, v16 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v13, v9, v13 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v11, v10, v11 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v17, v41, v17 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v15, v42, v15 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v14, v43, v14 +; TONGA-GCNTRACKERS-NEXT: v_add_f32_e32 v12, v5, v6 +; TONGA-GCNTRACKERS-NEXT: s_cbranch_scc1 .LBB4_1 +; TONGA-GCNTRACKERS-NEXT: ; %bb.2: ; %for.cond.cleanup26 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s4, v47, 0 +; TONGA-GCNTRACKERS-NEXT: s_lshl_b32 s0, s33, 8 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s10, v47, 6 +; TONGA-GCNTRACKERS-NEXT: s_add_i32 s0, s10, s0 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v0, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: s_mov_b32 s0, 0x5397829d +; TONGA-GCNTRACKERS-NEXT: v_mul_hi_u32 v0, v0, s0 +; TONGA-GCNTRACKERS-NEXT: s_mov_b32 s0, 0x31000 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, 0 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s6, v47, 2 +; TONGA-GCNTRACKERS-NEXT: v_lshrrev_b32_e32 v0, 4, v0 +; TONGA-GCNTRACKERS-NEXT: v_mul_lo_u32 v0, v0, s0 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s7, v47, 3 +; TONGA-GCNTRACKERS-NEXT: s_mov_b64 s[2:3], s[6:7] +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s3 +; TONGA-GCNTRACKERS-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1] +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x310 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v0, vcc, s2, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v5, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x348 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[5:6], v39 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v5, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: s_mov_b32 s0, 0 +; TONGA-GCNTRACKERS-NEXT: s_mov_b32 s1, s0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v8, s1 +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v7, s0 +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s1, 0x620 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx2 v[5:6], v[7:8] +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v5, vcc, s1, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s1, 0x930 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[5:6], v38 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v5, vcc, s1, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s1, 0xc40 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[5:6], v37 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v5, vcc, s1, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s1, 0xf50 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v9, vcc, s1, v0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[5:6], v36 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v10, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s0 +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s1, 0x1260 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx2 v[9:10], v[4:5] +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v5, vcc, s1, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s0 +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x1570 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx2 v[5:6], v[3:4] +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x1880 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v35 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x18b8 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v34 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x1b90 +; TONGA-GCNTRACKERS-NEXT: flat_store_dwordx2 v[2:3], v[7:8] +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x1ea0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v33 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x21b0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[0:1], v40 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v32 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x24c0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v31 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x27d0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v30 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x2ae0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v29 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x2df0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v28 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x3100 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v27 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x3410 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v26 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x3720 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v25 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x3a30 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v24 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x3d40 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v23 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x4050 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v22 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x4360 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v21 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x4670 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v20 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x4980 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v19 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x4c90 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v18 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x4fa0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v16 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x52b0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v13 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x55c0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v11 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x58d0 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v17 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, s0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v15 +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v2, vcc, 0x5be0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: v_add_u32_e32 v0, vcc, 0x5ef0, v0 +; TONGA-GCNTRACKERS-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s5, v47, 1 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s8, v47, 4 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s9, v47, 5 +; TONGA-GCNTRACKERS-NEXT: v_readlane_b32 s11, v47, 7 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[2:3], v14 +; TONGA-GCNTRACKERS-NEXT: flat_store_dword v[0:1], v12 +; TONGA-GCNTRACKERS-NEXT: s_endpgm +; +; GFX908-LABEL: excess_soft_clause_reg_pressure: +; GFX908: ; %bb.0: ; %entry +; GFX908-NEXT: s_load_dwordx8 s[52:59], s[4:5], 0x24 +; GFX908-NEXT: v_lshlrev_b32_e32 v1, 2, v0 +; GFX908-NEXT: s_movk_i32 s0, 0x200 +; GFX908-NEXT: v_mov_b32_e32 v5, 0 +; GFX908-NEXT: v_mov_b32_e32 v6, 0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: v_mov_b32_e32 v2, s57 +; GFX908-NEXT: v_add_co_u32_e32 v1, vcc, s56, v1 +; GFX908-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc +; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x188, v1 +; GFX908-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v2, vcc +; GFX908-NEXT: v_mov_b32_e32 v7, 0 +; GFX908-NEXT: v_mov_b32_e32 v8, 0 +; GFX908-NEXT: v_mov_b32_e32 v9, 0 +; GFX908-NEXT: v_mov_b32_e32 v14, 0 +; GFX908-NEXT: v_mov_b32_e32 v15, 0 +; GFX908-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-NEXT: v_mov_b32_e32 v17, 0 +; GFX908-NEXT: v_mov_b32_e32 v18, 0 +; GFX908-NEXT: v_mov_b32_e32 v19, 0 +; GFX908-NEXT: v_mov_b32_e32 v20, 0 +; GFX908-NEXT: v_mov_b32_e32 v21, 0 +; GFX908-NEXT: v_mov_b32_e32 v22, 0 +; GFX908-NEXT: v_mov_b32_e32 v23, 0 +; GFX908-NEXT: v_mov_b32_e32 v24, 0 +; GFX908-NEXT: v_mov_b32_e32 v25, 0 +; GFX908-NEXT: v_mov_b32_e32 v26, 0 +; GFX908-NEXT: v_mov_b32_e32 v27, 0 +; GFX908-NEXT: v_mov_b32_e32 v28, 0 +; GFX908-NEXT: v_mov_b32_e32 v29, 0 +; GFX908-NEXT: v_mov_b32_e32 v30, 0 +; GFX908-NEXT: v_mov_b32_e32 v31, 0 +; GFX908-NEXT: v_mov_b32_e32 v32, 0 +; GFX908-NEXT: v_mov_b32_e32 v33, 0 +; GFX908-NEXT: v_mov_b32_e32 v34, 0 +; GFX908-NEXT: v_mov_b32_e32 v10, 0 +; GFX908-NEXT: v_mov_b32_e32 v11, 0 +; GFX908-NEXT: v_mov_b32_e32 v12, 0 +; GFX908-NEXT: v_mov_b32_e32 v13, 0 +; GFX908-NEXT: v_mov_b32_e32 v1, 0 +; GFX908-NEXT: v_mov_b32_e32 v2, 0 +; GFX908-NEXT: .LBB4_1: ; %for.cond28.preheader +; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX908-NEXT: global_load_dword v38, v[3:4], off offset:-392 +; GFX908-NEXT: global_load_dword v37, v[3:4], off offset:-196 +; GFX908-NEXT: global_load_dword v36, v[3:4], off +; GFX908-NEXT: global_load_dword v35, v[3:4], off offset:196 +; GFX908-NEXT: s_load_dwordx16 s[36:51], s[52:53], 0x0 +; GFX908-NEXT: s_load_dwordx16 s[12:27], s[52:53], 0x40 +; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x310, v3 +; GFX908-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc +; GFX908-NEXT: s_waitcnt vmcnt(3) lgkmcnt(0) +; GFX908-NEXT: v_fmac_f32_e32 v34, s36, v38 +; GFX908-NEXT: v_fmac_f32_e32 v33, s37, v38 +; GFX908-NEXT: v_fmac_f32_e32 v32, s38, v38 +; GFX908-NEXT: v_fmac_f32_e32 v31, s39, v38 +; GFX908-NEXT: v_fmac_f32_e32 v30, s40, v38 +; GFX908-NEXT: v_fmac_f32_e32 v2, s41, v38 +; GFX908-NEXT: v_fmac_f32_e32 v1, s42, v38 +; GFX908-NEXT: v_fmac_f32_e32 v29, s43, v38 +; GFX908-NEXT: v_fmac_f32_e32 v28, s44, v38 +; GFX908-NEXT: v_fmac_f32_e32 v27, s45, v38 +; GFX908-NEXT: v_fmac_f32_e32 v26, s46, v38 +; GFX908-NEXT: v_fmac_f32_e32 v25, s47, v38 +; GFX908-NEXT: v_fmac_f32_e32 v24, s48, v38 +; GFX908-NEXT: v_fmac_f32_e32 v23, s49, v38 +; GFX908-NEXT: v_fmac_f32_e32 v22, s50, v38 +; GFX908-NEXT: v_fmac_f32_e32 v21, s51, v38 +; GFX908-NEXT: v_fmac_f32_e32 v20, s12, v38 +; GFX908-NEXT: v_fmac_f32_e32 v19, s13, v38 +; GFX908-NEXT: v_fmac_f32_e32 v18, s14, v38 +; GFX908-NEXT: v_fmac_f32_e32 v17, s15, v38 +; GFX908-NEXT: v_fmac_f32_e32 v16, s16, v38 +; GFX908-NEXT: v_fmac_f32_e32 v15, s17, v38 +; GFX908-NEXT: v_fmac_f32_e32 v14, s18, v38 +; GFX908-NEXT: v_fmac_f32_e32 v9, s19, v38 +; GFX908-NEXT: v_fmac_f32_e32 v8, s20, v38 +; GFX908-NEXT: v_fmac_f32_e32 v7, s21, v38 +; GFX908-NEXT: v_fmac_f32_e32 v6, s22, v38 +; GFX908-NEXT: v_fmac_f32_e32 v5, s23, v38 +; GFX908-NEXT: v_fmac_f32_e32 v10, s24, v38 +; GFX908-NEXT: v_fmac_f32_e32 v11, s25, v38 +; GFX908-NEXT: v_fmac_f32_e32 v12, s26, v38 +; GFX908-NEXT: v_fmac_f32_e32 v13, s27, v38 +; GFX908-NEXT: s_load_dwordx16 s[36:51], s[52:53], 0x1000 +; GFX908-NEXT: s_load_dwordx16 s[12:27], s[52:53], 0x1040 +; GFX908-NEXT: s_waitcnt vmcnt(2) lgkmcnt(0) +; GFX908-NEXT: v_fmac_f32_e32 v34, s36, v37 +; GFX908-NEXT: v_fmac_f32_e32 v33, s37, v37 +; GFX908-NEXT: v_fmac_f32_e32 v32, s38, v37 +; GFX908-NEXT: v_fmac_f32_e32 v31, s39, v37 +; GFX908-NEXT: v_fmac_f32_e32 v30, s40, v37 +; GFX908-NEXT: v_fmac_f32_e32 v2, s41, v37 +; GFX908-NEXT: v_fmac_f32_e32 v1, s42, v37 +; GFX908-NEXT: v_fmac_f32_e32 v29, s43, v37 +; GFX908-NEXT: v_fmac_f32_e32 v28, s44, v37 +; GFX908-NEXT: v_fmac_f32_e32 v27, s45, v37 +; GFX908-NEXT: v_fmac_f32_e32 v26, s46, v37 +; GFX908-NEXT: v_fmac_f32_e32 v25, s47, v37 +; GFX908-NEXT: v_fmac_f32_e32 v24, s48, v37 +; GFX908-NEXT: v_fmac_f32_e32 v23, s49, v37 +; GFX908-NEXT: v_fmac_f32_e32 v22, s50, v37 +; GFX908-NEXT: v_fmac_f32_e32 v21, s51, v37 +; GFX908-NEXT: v_fmac_f32_e32 v20, s12, v37 +; GFX908-NEXT: v_fmac_f32_e32 v19, s13, v37 +; GFX908-NEXT: v_fmac_f32_e32 v18, s14, v37 +; GFX908-NEXT: v_fmac_f32_e32 v17, s15, v37 +; GFX908-NEXT: v_fmac_f32_e32 v16, s16, v37 +; GFX908-NEXT: v_fmac_f32_e32 v15, s17, v37 +; GFX908-NEXT: v_fmac_f32_e32 v14, s18, v37 +; GFX908-NEXT: v_fmac_f32_e32 v9, s19, v37 +; GFX908-NEXT: v_fmac_f32_e32 v8, s20, v37 +; GFX908-NEXT: v_fmac_f32_e32 v7, s21, v37 +; GFX908-NEXT: v_fmac_f32_e32 v6, s22, v37 +; GFX908-NEXT: v_fmac_f32_e32 v5, s23, v37 +; GFX908-NEXT: v_fmac_f32_e32 v10, s24, v37 +; GFX908-NEXT: v_fmac_f32_e32 v11, s25, v37 +; GFX908-NEXT: v_fmac_f32_e32 v12, s26, v37 +; GFX908-NEXT: v_fmac_f32_e32 v13, s27, v37 +; GFX908-NEXT: s_load_dwordx16 s[36:51], s[52:53], 0x2000 +; GFX908-NEXT: s_load_dwordx16 s[12:27], s[52:53], 0x2040 +; GFX908-NEXT: s_waitcnt vmcnt(1) lgkmcnt(0) +; GFX908-NEXT: v_fmac_f32_e32 v34, s36, v36 +; GFX908-NEXT: v_fmac_f32_e32 v33, s37, v36 +; GFX908-NEXT: v_fmac_f32_e32 v32, s38, v36 +; GFX908-NEXT: v_fmac_f32_e32 v31, s39, v36 +; GFX908-NEXT: v_fmac_f32_e32 v30, s40, v36 +; GFX908-NEXT: v_fmac_f32_e32 v2, s41, v36 +; GFX908-NEXT: v_fmac_f32_e32 v1, s42, v36 +; GFX908-NEXT: v_fmac_f32_e32 v29, s43, v36 +; GFX908-NEXT: v_fmac_f32_e32 v28, s44, v36 +; GFX908-NEXT: v_fmac_f32_e32 v27, s45, v36 +; GFX908-NEXT: v_fmac_f32_e32 v26, s46, v36 +; GFX908-NEXT: v_fmac_f32_e32 v25, s47, v36 +; GFX908-NEXT: v_fmac_f32_e32 v24, s48, v36 +; GFX908-NEXT: v_fmac_f32_e32 v23, s49, v36 +; GFX908-NEXT: v_fmac_f32_e32 v22, s50, v36 +; GFX908-NEXT: v_fmac_f32_e32 v21, s51, v36 +; GFX908-NEXT: v_fmac_f32_e32 v20, s12, v36 +; GFX908-NEXT: v_fmac_f32_e32 v19, s13, v36 +; GFX908-NEXT: v_fmac_f32_e32 v18, s14, v36 +; GFX908-NEXT: v_fmac_f32_e32 v17, s15, v36 +; GFX908-NEXT: v_fmac_f32_e32 v16, s16, v36 +; GFX908-NEXT: v_fmac_f32_e32 v15, s17, v36 +; GFX908-NEXT: v_fmac_f32_e32 v14, s18, v36 +; GFX908-NEXT: v_fmac_f32_e32 v9, s19, v36 +; GFX908-NEXT: v_fmac_f32_e32 v8, s20, v36 +; GFX908-NEXT: v_fmac_f32_e32 v7, s21, v36 +; GFX908-NEXT: v_fmac_f32_e32 v6, s22, v36 +; GFX908-NEXT: v_fmac_f32_e32 v5, s23, v36 +; GFX908-NEXT: v_fmac_f32_e32 v10, s24, v36 +; GFX908-NEXT: v_fmac_f32_e32 v11, s25, v36 +; GFX908-NEXT: v_fmac_f32_e32 v12, s26, v36 +; GFX908-NEXT: v_fmac_f32_e32 v13, s27, v36 +; GFX908-NEXT: s_load_dwordx16 s[36:51], s[52:53], 0x3000 +; GFX908-NEXT: s_load_dwordx16 s[12:27], s[52:53], 0x3040 +; GFX908-NEXT: s_add_u32 s52, s52, 0x4000 +; GFX908-NEXT: s_addc_u32 s53, s53, 0 +; GFX908-NEXT: s_add_i32 s0, s0, -1 +; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX908-NEXT: v_fmac_f32_e32 v34, s36, v35 +; GFX908-NEXT: v_fmac_f32_e32 v33, s37, v35 +; GFX908-NEXT: v_fmac_f32_e32 v32, s38, v35 +; GFX908-NEXT: v_fmac_f32_e32 v31, s39, v35 +; GFX908-NEXT: v_fmac_f32_e32 v30, s40, v35 +; GFX908-NEXT: v_fmac_f32_e32 v2, s41, v35 +; GFX908-NEXT: v_fmac_f32_e32 v1, s42, v35 +; GFX908-NEXT: v_fmac_f32_e32 v29, s43, v35 +; GFX908-NEXT: v_fmac_f32_e32 v28, s44, v35 +; GFX908-NEXT: v_fmac_f32_e32 v27, s45, v35 +; GFX908-NEXT: v_fmac_f32_e32 v26, s46, v35 +; GFX908-NEXT: v_fmac_f32_e32 v25, s47, v35 +; GFX908-NEXT: v_fmac_f32_e32 v24, s48, v35 +; GFX908-NEXT: v_fmac_f32_e32 v23, s49, v35 +; GFX908-NEXT: v_fmac_f32_e32 v22, s50, v35 +; GFX908-NEXT: v_fmac_f32_e32 v21, s51, v35 +; GFX908-NEXT: v_fmac_f32_e32 v20, s12, v35 +; GFX908-NEXT: v_fmac_f32_e32 v19, s13, v35 +; GFX908-NEXT: v_fmac_f32_e32 v18, s14, v35 +; GFX908-NEXT: v_fmac_f32_e32 v17, s15, v35 +; GFX908-NEXT: v_fmac_f32_e32 v16, s16, v35 +; GFX908-NEXT: v_fmac_f32_e32 v15, s17, v35 +; GFX908-NEXT: v_fmac_f32_e32 v14, s18, v35 +; GFX908-NEXT: v_fmac_f32_e32 v9, s19, v35 +; GFX908-NEXT: v_fmac_f32_e32 v8, s20, v35 +; GFX908-NEXT: v_fmac_f32_e32 v7, s21, v35 +; GFX908-NEXT: v_fmac_f32_e32 v6, s22, v35 +; GFX908-NEXT: v_fmac_f32_e32 v5, s23, v35 +; GFX908-NEXT: v_fmac_f32_e32 v10, s24, v35 +; GFX908-NEXT: v_fmac_f32_e32 v11, s25, v35 +; GFX908-NEXT: v_fmac_f32_e32 v12, s26, v35 +; GFX908-NEXT: v_fmac_f32_e32 v13, s27, v35 +; GFX908-NEXT: s_cmp_lg_u32 s0, 0 +; GFX908-NEXT: s_cbranch_scc1 .LBB4_1 +; GFX908-NEXT: ; %bb.2: ; %for.cond.cleanup26 +; GFX908-NEXT: s_lshl_b32 s0, s8, 8 +; GFX908-NEXT: s_add_i32 s0, s58, s0 +; GFX908-NEXT: v_add_u32_e32 v0, s0, v0 +; GFX908-NEXT: s_mov_b32 s0, 0x5397829d +; GFX908-NEXT: v_mul_hi_u32 v0, v0, s0 +; GFX908-NEXT: s_mov_b32 s0, 0x31000 +; GFX908-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-NEXT: s_movk_i32 s1, 0x310 +; GFX908-NEXT: v_lshrrev_b32_e32 v0, 4, v0 +; GFX908-NEXT: v_mul_lo_u32 v3, v0, s0 +; GFX908-NEXT: v_mov_b32_e32 v0, s55 +; GFX908-NEXT: s_mov_b32 s0, 0 +; GFX908-NEXT: v_lshlrev_b64 v[3:4], 2, v[3:4] +; GFX908-NEXT: v_add_co_u32_e32 v35, vcc, s54, v3 +; GFX908-NEXT: v_addc_co_u32_e32 v36, vcc, v0, v4, vcc +; GFX908-NEXT: v_mov_b32_e32 v3, s0 +; GFX908-NEXT: global_store_dwordx2 v[35:36], v[2:3], off offset:3920 +; GFX908-NEXT: global_store_dword v[35:36], v34, off +; GFX908-NEXT: global_store_dword v[35:36], v33, off offset:784 +; GFX908-NEXT: global_store_dword v[35:36], v32, off offset:1568 +; GFX908-NEXT: global_store_dword v[35:36], v31, off offset:2352 +; GFX908-NEXT: global_store_dword v[35:36], v30, off offset:3136 +; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, s1, v35 +; GFX908-NEXT: s_mov_b32 s1, s0 +; GFX908-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v36, vcc +; GFX908-NEXT: v_mov_b32_e32 v31, s1 +; GFX908-NEXT: v_mov_b32_e32 v30, s0 +; GFX908-NEXT: s_movk_i32 s1, 0x620 +; GFX908-NEXT: v_add_co_u32_e32 v32, vcc, s1, v35 +; GFX908-NEXT: v_addc_co_u32_e32 v33, vcc, 0, v36, vcc +; GFX908-NEXT: s_movk_i32 s1, 0x930 +; GFX908-NEXT: v_add_co_u32_e32 v37, vcc, s1, v35 +; GFX908-NEXT: v_addc_co_u32_e32 v38, vcc, 0, v36, vcc +; GFX908-NEXT: s_movk_i32 s1, 0xc40 +; GFX908-NEXT: v_add_co_u32_e32 v39, vcc, s1, v35 +; GFX908-NEXT: v_addc_co_u32_e32 v40, vcc, 0, v36, vcc +; GFX908-NEXT: s_movk_i32 s1, 0xf50 +; GFX908-NEXT: v_add_co_u32_e32 v34, vcc, s1, v35 +; GFX908-NEXT: global_store_dwordx2 v[35:36], v[30:31], off offset:840 +; GFX908-NEXT: v_addc_co_u32_e32 v35, vcc, 0, v36, vcc +; GFX908-NEXT: v_mov_b32_e32 v2, s0 +; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, s1, v3 +; GFX908-NEXT: global_store_dwordx2 v[3:4], v[1:2], off offset:3920 +; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v4, vcc +; GFX908-NEXT: v_add_co_u32_e32 v2, vcc, s1, v32 +; GFX908-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v33, vcc +; GFX908-NEXT: global_store_dwordx2 v[37:38], v[30:31], off offset:3976 +; GFX908-NEXT: global_store_dword v[37:38], v28, off offset:3920 +; GFX908-NEXT: v_add_co_u32_e32 v28, vcc, s1, v37 +; GFX908-NEXT: global_store_dword v[32:33], v29, off offset:3920 +; GFX908-NEXT: v_addc_co_u32_e32 v29, vcc, 0, v38, vcc +; GFX908-NEXT: v_add_co_u32_e32 v30, vcc, s1, v39 +; GFX908-NEXT: v_addc_co_u32_e32 v31, vcc, 0, v40, vcc +; GFX908-NEXT: global_store_dword v[39:40], v27, off offset:3920 +; GFX908-NEXT: global_store_dword v[34:35], v26, off offset:3920 +; GFX908-NEXT: v_add_co_u32_e32 v26, vcc, s1, v34 +; GFX908-NEXT: v_addc_co_u32_e32 v27, vcc, 0, v35, vcc +; GFX908-NEXT: global_store_dword v[0:1], v25, off offset:3920 +; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, s1, v0 +; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GFX908-NEXT: global_store_dword v[2:3], v24, off offset:3920 +; GFX908-NEXT: v_add_co_u32_e32 v2, vcc, s1, v2 +; GFX908-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc +; GFX908-NEXT: global_store_dword v[28:29], v23, off offset:3920 +; GFX908-NEXT: v_add_co_u32_e32 v23, vcc, s1, v28 +; GFX908-NEXT: v_addc_co_u32_e32 v24, vcc, 0, v29, vcc +; GFX908-NEXT: v_add_co_u32_e32 v28, vcc, s1, v30 +; GFX908-NEXT: v_addc_co_u32_e32 v29, vcc, 0, v31, vcc +; GFX908-NEXT: global_store_dword v[26:27], v21, off offset:3920 +; GFX908-NEXT: v_add_co_u32_e32 v21, vcc, s1, v26 +; GFX908-NEXT: global_store_dword v[30:31], v22, off offset:3920 +; GFX908-NEXT: v_addc_co_u32_e32 v22, vcc, 0, v27, vcc +; GFX908-NEXT: global_store_dword v[0:1], v20, off offset:3920 +; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, s1, v0 +; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GFX908-NEXT: global_store_dword v[2:3], v19, off offset:3920 +; GFX908-NEXT: v_add_co_u32_e32 v2, vcc, s1, v2 +; GFX908-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc +; GFX908-NEXT: global_store_dword v[23:24], v18, off offset:3920 +; GFX908-NEXT: v_add_co_u32_e32 v18, vcc, s1, v23 +; GFX908-NEXT: v_addc_co_u32_e32 v19, vcc, 0, v24, vcc +; GFX908-NEXT: v_add_co_u32_e32 v23, vcc, s1, v28 +; GFX908-NEXT: v_addc_co_u32_e32 v24, vcc, 0, v29, vcc +; GFX908-NEXT: v_add_co_u32_e32 v25, vcc, s1, v21 +; GFX908-NEXT: v_addc_co_u32_e32 v26, vcc, 0, v22, vcc +; GFX908-NEXT: global_store_dword v[28:29], v17, off offset:3920 +; GFX908-NEXT: global_store_dword v[21:22], v16, off offset:3920 +; GFX908-NEXT: v_add_co_u32_e32 v16, vcc, s1, v0 +; GFX908-NEXT: v_addc_co_u32_e32 v17, vcc, 0, v1, vcc +; GFX908-NEXT: global_store_dword v[0:1], v15, off offset:3920 +; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, s1, v2 +; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc +; GFX908-NEXT: global_store_dword v[2:3], v14, off offset:3920 +; GFX908-NEXT: v_add_co_u32_e32 v2, vcc, s1, v18 +; GFX908-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v19, vcc +; GFX908-NEXT: v_add_co_u32_e32 v14, vcc, s1, v23 +; GFX908-NEXT: v_addc_co_u32_e32 v15, vcc, 0, v24, vcc +; GFX908-NEXT: global_store_dword v[23:24], v8, off offset:3920 +; GFX908-NEXT: v_add_co_u32_e32 v8, vcc, 0xf50, v25 +; GFX908-NEXT: global_store_dword v[18:19], v9, off offset:3920 +; GFX908-NEXT: v_addc_co_u32_e32 v9, vcc, 0, v26, vcc +; GFX908-NEXT: v_add_co_u32_e32 v18, vcc, 0xf50, v16 +; GFX908-NEXT: global_store_dword v[25:26], v7, off offset:3920 +; GFX908-NEXT: v_addc_co_u32_e32 v19, vcc, 0, v17, vcc +; GFX908-NEXT: global_store_dword v[16:17], v6, off offset:3920 +; GFX908-NEXT: global_store_dword v[0:1], v5, off offset:3920 +; GFX908-NEXT: global_store_dword v[2:3], v10, off offset:3920 +; GFX908-NEXT: global_store_dword v[14:15], v11, off offset:3920 +; GFX908-NEXT: global_store_dword v[8:9], v12, off offset:3920 +; GFX908-NEXT: global_store_dword v[18:19], v13, off offset:3920 +; GFX908-NEXT: s_endpgm +; +; GFX908-GCNTRACKERS-LABEL: excess_soft_clause_reg_pressure: +; GFX908-GCNTRACKERS: ; %bb.0: ; %entry +; GFX908-GCNTRACKERS-NEXT: s_load_dwordx8 s[52:59], s[4:5], 0x24 +; GFX908-GCNTRACKERS-NEXT: v_lshlrev_b32_e32 v2, 2, v0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, 0 +; GFX908-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x200 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, 0 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s57 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v2, vcc, s56, v2 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v3, vcc +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v3, vcc, 0x188, v2 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v6, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v7, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v8, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v9, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v14, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v15, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v17, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v18, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v19, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v20, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v21, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v22, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v23, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v24, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v25, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v26, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v27, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v28, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v29, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v30, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v31, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v32, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v33, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v34, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v10, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v11, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v12, 0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v13, 0 +; GFX908-GCNTRACKERS-NEXT: .LBB4_1: ; %for.cond28.preheader +; GFX908-GCNTRACKERS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX908-GCNTRACKERS-NEXT: global_load_dword v38, v[3:4], off offset:-392 +; GFX908-GCNTRACKERS-NEXT: global_load_dword v37, v[3:4], off offset:-196 +; GFX908-GCNTRACKERS-NEXT: global_load_dword v36, v[3:4], off +; GFX908-GCNTRACKERS-NEXT: global_load_dword v35, v[3:4], off offset:196 +; GFX908-GCNTRACKERS-NEXT: s_load_dwordx16 s[36:51], s[52:53], 0x0 +; GFX908-GCNTRACKERS-NEXT: s_load_dwordx16 s[12:27], s[52:53], 0x40 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v3, vcc, 0x310, v3 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(3) lgkmcnt(0) +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v34, s36, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v33, s37, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v32, s38, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v31, s39, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v30, s40, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v2, s41, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v1, s42, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v29, s43, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v28, s44, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v27, s45, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v26, s46, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v25, s47, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v24, s48, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v23, s49, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v22, s50, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v21, s51, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v20, s12, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v19, s13, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v18, s14, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v17, s15, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v16, s16, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v15, s17, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v14, s18, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v9, s19, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v8, s20, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v7, s21, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v6, s22, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v5, s23, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v10, s24, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v11, s25, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v12, s26, v38 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v13, s27, v38 +; GFX908-GCNTRACKERS-NEXT: s_load_dwordx16 s[36:51], s[52:53], 0x1000 +; GFX908-GCNTRACKERS-NEXT: s_load_dwordx16 s[12:27], s[52:53], 0x1040 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(2) lgkmcnt(0) +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v34, s36, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v33, s37, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v32, s38, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v31, s39, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v30, s40, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v2, s41, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v1, s42, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v29, s43, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v28, s44, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v27, s45, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v26, s46, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v25, s47, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v24, s48, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v23, s49, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v22, s50, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v21, s51, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v20, s12, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v19, s13, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v18, s14, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v17, s15, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v16, s16, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v15, s17, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v14, s18, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v9, s19, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v8, s20, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v7, s21, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v6, s22, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v5, s23, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v10, s24, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v11, s25, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v12, s26, v37 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v13, s27, v37 +; GFX908-GCNTRACKERS-NEXT: s_load_dwordx16 s[36:51], s[52:53], 0x2000 +; GFX908-GCNTRACKERS-NEXT: s_load_dwordx16 s[12:27], s[52:53], 0x2040 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(1) lgkmcnt(0) +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v34, s36, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v33, s37, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v32, s38, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v31, s39, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v30, s40, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v2, s41, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v1, s42, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v29, s43, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v28, s44, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v27, s45, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v26, s46, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v25, s47, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v24, s48, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v23, s49, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v22, s50, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v21, s51, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v20, s12, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v19, s13, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v18, s14, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v17, s15, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v16, s16, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v15, s17, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v14, s18, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v9, s19, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v8, s20, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v7, s21, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v6, s22, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v5, s23, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v10, s24, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v11, s25, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v12, s26, v36 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v13, s27, v36 +; GFX908-GCNTRACKERS-NEXT: s_load_dwordx16 s[36:51], s[52:53], 0x3000 +; GFX908-GCNTRACKERS-NEXT: s_load_dwordx16 s[12:27], s[52:53], 0x3040 +; GFX908-GCNTRACKERS-NEXT: s_add_u32 s52, s52, 0x4000 +; GFX908-GCNTRACKERS-NEXT: s_addc_u32 s53, s53, 0 +; GFX908-GCNTRACKERS-NEXT: s_add_i32 s0, s0, -1 +; GFX908-GCNTRACKERS-NEXT: s_cmp_lg_u32 s0, 0 +; GFX908-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v34, s36, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v33, s37, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v32, s38, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v31, s39, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v30, s40, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v2, s41, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v1, s42, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v29, s43, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v28, s44, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v27, s45, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v26, s46, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v25, s47, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v24, s48, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v23, s49, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v22, s50, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v21, s51, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v20, s12, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v19, s13, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v18, s14, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v17, s15, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v16, s16, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v15, s17, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v14, s18, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v9, s19, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v8, s20, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v7, s21, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v6, s22, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v5, s23, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v10, s24, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v11, s25, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v12, s26, v35 +; GFX908-GCNTRACKERS-NEXT: v_fmac_f32_e32 v13, s27, v35 +; GFX908-GCNTRACKERS-NEXT: s_cbranch_scc1 .LBB4_1 +; GFX908-GCNTRACKERS-NEXT: ; %bb.2: ; %for.cond.cleanup26 +; GFX908-GCNTRACKERS-NEXT: s_lshl_b32 s0, s8, 8 +; GFX908-GCNTRACKERS-NEXT: s_add_i32 s0, s58, s0 +; GFX908-GCNTRACKERS-NEXT: v_add_u32_e32 v0, s0, v0 +; GFX908-GCNTRACKERS-NEXT: s_mov_b32 s0, 0x5397829d +; GFX908-GCNTRACKERS-NEXT: v_mul_hi_u32 v0, v0, s0 +; GFX908-GCNTRACKERS-NEXT: s_mov_b32 s0, 0x31000 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-GCNTRACKERS-NEXT: s_movk_i32 s1, 0x310 +; GFX908-GCNTRACKERS-NEXT: v_lshrrev_b32_e32 v0, 4, v0 +; GFX908-GCNTRACKERS-NEXT: v_mul_lo_u32 v3, v0, s0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s55 +; GFX908-GCNTRACKERS-NEXT: s_mov_b32 s0, 0 +; GFX908-GCNTRACKERS-NEXT: v_lshlrev_b64 v[3:4], 2, v[3:4] +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v35, vcc, s54, v3 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v36, vcc, v0, v4, vcc +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, s0 +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx2 v[35:36], v[2:3], off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v3, vcc, s1, v35 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v36, vcc +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s0 +; GFX908-GCNTRACKERS-NEXT: s_mov_b32 s1, s0 +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx2 v[3:4], v[1:2], off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v0, s0 +; GFX908-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x930 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v37, vcc, s0, v35 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v38, vcc, 0, v36, vcc +; GFX908-GCNTRACKERS-NEXT: s_movk_i32 s0, 0x620 +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx2 v[35:36], v[0:1], off offset:840 +; GFX908-GCNTRACKERS-NEXT: global_store_dwordx2 v[37:38], v[0:1], off offset:3976 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[35:36], v34, off +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[35:36], v33, off offset:784 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[35:36], v32, off offset:1568 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[35:36], v31, off offset:2352 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[35:36], v30, off offset:3136 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v0, vcc, s0, v35 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v36, vcc +; GFX908-GCNTRACKERS-NEXT: s_movk_i32 s0, 0xc40 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[0:1], v29, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[37:38], v28, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v28, vcc, s0, v35 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v29, vcc, 0, v36, vcc +; GFX908-GCNTRACKERS-NEXT: s_movk_i32 s0, 0xf50 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v30, vcc, s0, v35 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v31, vcc, 0, v36, vcc +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v2, vcc, s0, v3 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v4, vcc +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[28:29], v27, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[0:1], v24, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v24, vcc, s0, v37 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[2:3], v25, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v38, vcc +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[30:31], v26, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v26, vcc, s0, v28 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v27, vcc, 0, v29, vcc +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[26:27], v22, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v22, vcc, s0, v30 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[24:25], v23, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v23, vcc, 0, v31, vcc +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v2, vcc, s0, v2 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[2:3], v20, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v20, vcc, s0, v24 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[22:23], v21, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v21, vcc, 0, v25, vcc +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v24, vcc, s0, v26 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v27, vcc +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v22, vcc, s0, v22 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v23, vcc, 0, v23, vcc +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v2, vcc, s0, v2 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[0:1], v19, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[20:21], v18, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v18, vcc, s0, v20 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v19, vcc, 0, v21, vcc +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v20, vcc, s0, v24 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v21, vcc, 0, v25, vcc +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[24:25], v17, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v24, vcc, s0, v22 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v23, vcc +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[22:23], v16, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v16, vcc, s0, v2 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v17, vcc, 0, v3, vcc +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[2:3], v15, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v2, vcc, s0, v0 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[0:1], v14, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v0, vcc, s0, v18 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v19, vcc +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v14, vcc, s0, v20 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v15, vcc, 0, v21, vcc +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[20:21], v8, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v8, vcc, 0xf50, v24 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[18:19], v9, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v9, vcc, 0, v25, vcc +; GFX908-GCNTRACKERS-NEXT: v_add_co_u32_e32 v18, vcc, 0xf50, v16 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[24:25], v7, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: v_addc_co_u32_e32 v19, vcc, 0, v17, vcc +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[16:17], v6, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[2:3], v5, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[0:1], v10, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[14:15], v11, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[8:9], v12, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: global_store_dword v[18:19], v13, off offset:3920 +; GFX908-GCNTRACKERS-NEXT: s_endpgm +; +; GENERIC-LABEL: excess_soft_clause_reg_pressure: +; GENERIC: ; %bb.0: ; %entry +; GENERIC-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9 +; GENERIC-NEXT: v_lshlrev_b32_e32 v1, 2, v0 +; GENERIC-NEXT: s_mov_b32 s30, 0 +; GENERIC-NEXT: s_mov_b32 s31, 0xf000 +; GENERIC-NEXT: s_waitcnt lgkmcnt(0) +; GENERIC-NEXT: s_movk_i32 s7, 0x3000 +; GENERIC-NEXT: v_mov_b32_e32 v2, s5 +; GENERIC-NEXT: v_add_i32_e32 v1, vcc, s4, v1 +; GENERIC-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc +; GENERIC-NEXT: s_movk_i32 s4, 0x1000 +; GENERIC-NEXT: s_movk_i32 s5, 0x2000 +; GENERIC-NEXT: s_movk_i32 s9, 0x1040 +; GENERIC-NEXT: s_movk_i32 s10, 0x2040 +; GENERIC-NEXT: s_movk_i32 s11, 0x3040 +; GENERIC-NEXT: s_movk_i32 s33, 0x310 +; GENERIC-NEXT: v_mov_b32_e32 v6, 0 +; GENERIC-NEXT: v_mov_b32_e32 v7, 0 +; GENERIC-NEXT: v_mov_b32_e32 v8, 0 +; GENERIC-NEXT: v_mov_b32_e32 v9, 0 +; GENERIC-NEXT: v_mov_b32_e32 v10, 0 +; GENERIC-NEXT: v_mov_b32_e32 v11, 0 +; GENERIC-NEXT: v_mov_b32_e32 v12, 0 +; GENERIC-NEXT: v_mov_b32_e32 v16, 0 +; GENERIC-NEXT: v_mov_b32_e32 v18, 0 +; GENERIC-NEXT: v_mov_b32_e32 v19, 0 +; GENERIC-NEXT: v_mov_b32_e32 v20, 0 +; GENERIC-NEXT: v_mov_b32_e32 v21, 0 +; GENERIC-NEXT: v_mov_b32_e32 v22, 0 +; GENERIC-NEXT: v_mov_b32_e32 v23, 0 +; GENERIC-NEXT: v_mov_b32_e32 v24, 0 +; GENERIC-NEXT: v_mov_b32_e32 v25, 0 +; GENERIC-NEXT: v_mov_b32_e32 v26, 0 +; GENERIC-NEXT: s_mov_b32 s34, s30 +; GENERIC-NEXT: v_mov_b32_e32 v27, 0 +; GENERIC-NEXT: v_mov_b32_e32 v28, 0 +; GENERIC-NEXT: v_mov_b32_e32 v29, 0 +; GENERIC-NEXT: v_mov_b32_e32 v30, 0 +; GENERIC-NEXT: v_mov_b32_e32 v32, 0 +; GENERIC-NEXT: v_mov_b32_e32 v33, 0 +; GENERIC-NEXT: v_mov_b32_e32 v34, 0 +; GENERIC-NEXT: v_mov_b32_e32 v35, 0 +; GENERIC-NEXT: v_mov_b32_e32 v36, 0 +; GENERIC-NEXT: v_mov_b32_e32 v17, 0 +; GENERIC-NEXT: v_mov_b32_e32 v15, 0 +; GENERIC-NEXT: v_mov_b32_e32 v14, 0 +; GENERIC-NEXT: v_mov_b32_e32 v13, 0 +; GENERIC-NEXT: v_mov_b32_e32 v3, 0 +; GENERIC-NEXT: v_mov_b32_e32 v4, 0 +; GENERIC-NEXT: .LBB4_1: ; %for.cond28.preheader +; GENERIC-NEXT: ; =>This Inner Loop Header: Depth=1 +; GENERIC-NEXT: s_mov_b32 s28, s30 +; GENERIC-NEXT: s_mov_b32 s29, s30 +; GENERIC-NEXT: buffer_load_dword v38, v[1:2], s[28:31], 0 addr64 +; GENERIC-NEXT: buffer_load_dword v37, v[1:2], s[28:31], 0 addr64 offset:196 +; GENERIC-NEXT: buffer_load_dword v31, v[1:2], s[28:31], 0 addr64 offset:392 +; GENERIC-NEXT: buffer_load_dword v5, v[1:2], s[28:31], 0 addr64 offset:588 +; GENERIC-NEXT: s_load_dwordx16 s[36:51], s[0:1], 0x0 +; GENERIC-NEXT: s_load_dwordx16 s[12:27], s[0:1], 0x10 +; GENERIC-NEXT: v_add_i32_e32 v1, vcc, s33, v1 +; GENERIC-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc +; GENERIC-NEXT: s_waitcnt vmcnt(3) lgkmcnt(0) +; GENERIC-NEXT: v_fma_f32 v4, v38, s41, v4 +; GENERIC-NEXT: v_fma_f32 v3, v38, s42, v3 +; GENERIC-NEXT: v_fma_f32 v36, v38, s36, v36 +; GENERIC-NEXT: v_fma_f32 v35, v38, s37, v35 +; GENERIC-NEXT: v_fma_f32 v34, v38, s38, v34 +; GENERIC-NEXT: v_fma_f32 v33, v38, s39, v33 +; GENERIC-NEXT: v_fma_f32 v32, v38, s40, v32 +; GENERIC-NEXT: v_fma_f32 v30, v38, s43, v30 +; GENERIC-NEXT: v_fma_f32 v29, v38, s44, v29 +; GENERIC-NEXT: v_fma_f32 v28, v38, s45, v28 +; GENERIC-NEXT: v_fma_f32 v27, v38, s46, v27 +; GENERIC-NEXT: v_fma_f32 v26, v38, s47, v26 +; GENERIC-NEXT: v_fma_f32 v25, v38, s48, v25 +; GENERIC-NEXT: v_fma_f32 v24, v38, s49, v24 +; GENERIC-NEXT: v_fma_f32 v23, v38, s50, v23 +; GENERIC-NEXT: v_fma_f32 v22, v38, s51, v22 +; GENERIC-NEXT: v_fma_f32 v21, v38, s12, v21 +; GENERIC-NEXT: v_fma_f32 v20, v38, s13, v20 +; GENERIC-NEXT: v_fma_f32 v19, v38, s14, v19 +; GENERIC-NEXT: v_fma_f32 v18, v38, s15, v18 +; GENERIC-NEXT: v_fma_f32 v16, v38, s16, v16 +; GENERIC-NEXT: v_fma_f32 v12, v38, s17, v12 +; GENERIC-NEXT: v_fma_f32 v11, v38, s18, v11 +; GENERIC-NEXT: v_fma_f32 v10, v38, s19, v10 +; GENERIC-NEXT: v_fma_f32 v9, v38, s20, v9 +; GENERIC-NEXT: v_fma_f32 v8, v38, s21, v8 +; GENERIC-NEXT: v_fma_f32 v7, v38, s22, v7 +; GENERIC-NEXT: v_fma_f32 v6, v38, s23, v6 +; GENERIC-NEXT: v_fma_f32 v17, v38, s24, v17 +; GENERIC-NEXT: v_fma_f32 v15, v38, s25, v15 +; GENERIC-NEXT: v_fma_f32 v14, v38, s26, v14 +; GENERIC-NEXT: v_fma_f32 v13, v38, s27, v13 +; GENERIC-NEXT: s_load_dwordx16 s[36:51], s[0:1], s4 +; GENERIC-NEXT: s_load_dwordx16 s[12:27], s[0:1], s9 +; GENERIC-NEXT: s_waitcnt vmcnt(2) lgkmcnt(0) +; GENERIC-NEXT: v_fma_f32 v36, v37, s36, v36 +; GENERIC-NEXT: v_fma_f32 v35, v37, s37, v35 +; GENERIC-NEXT: v_fma_f32 v34, v37, s38, v34 +; GENERIC-NEXT: v_fma_f32 v33, v37, s39, v33 +; GENERIC-NEXT: v_fma_f32 v32, v37, s40, v32 +; GENERIC-NEXT: v_fma_f32 v4, v37, s41, v4 +; GENERIC-NEXT: v_fma_f32 v3, v37, s42, v3 +; GENERIC-NEXT: v_fma_f32 v30, v37, s43, v30 +; GENERIC-NEXT: v_fma_f32 v29, v37, s44, v29 +; GENERIC-NEXT: v_fma_f32 v28, v37, s45, v28 +; GENERIC-NEXT: v_fma_f32 v27, v37, s46, v27 +; GENERIC-NEXT: v_fma_f32 v26, v37, s47, v26 +; GENERIC-NEXT: v_fma_f32 v25, v37, s48, v25 +; GENERIC-NEXT: v_fma_f32 v24, v37, s49, v24 +; GENERIC-NEXT: v_fma_f32 v23, v37, s50, v23 +; GENERIC-NEXT: v_fma_f32 v22, v37, s51, v22 +; GENERIC-NEXT: v_fma_f32 v21, v37, s12, v21 +; GENERIC-NEXT: v_fma_f32 v20, v37, s13, v20 +; GENERIC-NEXT: v_fma_f32 v19, v37, s14, v19 +; GENERIC-NEXT: v_fma_f32 v18, v37, s15, v18 +; GENERIC-NEXT: v_fma_f32 v16, v37, s16, v16 +; GENERIC-NEXT: v_fma_f32 v12, v37, s17, v12 +; GENERIC-NEXT: v_fma_f32 v11, v37, s18, v11 +; GENERIC-NEXT: v_fma_f32 v10, v37, s19, v10 +; GENERIC-NEXT: v_fma_f32 v9, v37, s20, v9 +; GENERIC-NEXT: v_fma_f32 v8, v37, s21, v8 +; GENERIC-NEXT: v_fma_f32 v7, v37, s22, v7 +; GENERIC-NEXT: v_fma_f32 v6, v37, s23, v6 +; GENERIC-NEXT: v_fma_f32 v17, v37, s24, v17 +; GENERIC-NEXT: v_fma_f32 v15, v37, s25, v15 +; GENERIC-NEXT: v_fma_f32 v14, v37, s26, v14 +; GENERIC-NEXT: v_fma_f32 v13, v37, s27, v13 +; GENERIC-NEXT: s_load_dwordx16 s[52:67], s[0:1], s5 +; GENERIC-NEXT: s_load_dwordx16 s[12:27], s[0:1], s10 +; GENERIC-NEXT: s_load_dwordx16 s[36:51], s[0:1], s7 +; GENERIC-NEXT: s_waitcnt vmcnt(1) lgkmcnt(0) +; GENERIC-NEXT: v_fma_f32 v36, v31, s52, v36 +; GENERIC-NEXT: v_fma_f32 v35, v31, s53, v35 +; GENERIC-NEXT: v_fma_f32 v34, v31, s54, v34 +; GENERIC-NEXT: v_fma_f32 v33, v31, s55, v33 +; GENERIC-NEXT: v_fma_f32 v32, v31, s56, v32 +; GENERIC-NEXT: v_fma_f32 v4, v31, s57, v4 +; GENERIC-NEXT: v_fma_f32 v3, v31, s58, v3 +; GENERIC-NEXT: v_fma_f32 v30, v31, s59, v30 +; GENERIC-NEXT: v_fma_f32 v29, v31, s60, v29 +; GENERIC-NEXT: v_fma_f32 v28, v31, s61, v28 +; GENERIC-NEXT: v_fma_f32 v27, v31, s62, v27 +; GENERIC-NEXT: v_fma_f32 v26, v31, s63, v26 +; GENERIC-NEXT: v_fma_f32 v25, v31, s64, v25 +; GENERIC-NEXT: v_fma_f32 v24, v31, s65, v24 +; GENERIC-NEXT: v_fma_f32 v23, v31, s66, v23 +; GENERIC-NEXT: v_fma_f32 v22, v31, s67, v22 +; GENERIC-NEXT: s_load_dwordx16 s[52:67], s[0:1], s11 +; GENERIC-NEXT: s_add_u32 s0, s0, 0x4000 +; GENERIC-NEXT: s_addc_u32 s1, s1, 0 +; GENERIC-NEXT: s_add_i32 s34, s34, 1 +; GENERIC-NEXT: v_fma_f32 v21, v31, s12, v21 +; GENERIC-NEXT: v_fma_f32 v20, v31, s13, v20 +; GENERIC-NEXT: v_fma_f32 v19, v31, s14, v19 +; GENERIC-NEXT: v_fma_f32 v18, v31, s15, v18 +; GENERIC-NEXT: v_fma_f32 v16, v31, s16, v16 +; GENERIC-NEXT: v_fma_f32 v12, v31, s17, v12 +; GENERIC-NEXT: v_fma_f32 v11, v31, s18, v11 +; GENERIC-NEXT: v_fma_f32 v10, v31, s19, v10 +; GENERIC-NEXT: v_fma_f32 v9, v31, s20, v9 +; GENERIC-NEXT: v_fma_f32 v8, v31, s21, v8 +; GENERIC-NEXT: v_fma_f32 v7, v31, s22, v7 +; GENERIC-NEXT: v_fma_f32 v6, v31, s23, v6 +; GENERIC-NEXT: v_fma_f32 v17, v31, s24, v17 +; GENERIC-NEXT: v_fma_f32 v15, v31, s25, v15 +; GENERIC-NEXT: v_fma_f32 v14, v31, s26, v14 +; GENERIC-NEXT: v_fma_f32 v13, v31, s27, v13 +; GENERIC-NEXT: s_cmpk_lg_i32 s34, 0x200 +; GENERIC-NEXT: s_waitcnt vmcnt(0) +; GENERIC-NEXT: v_fma_f32 v36, v5, s36, v36 +; GENERIC-NEXT: v_fma_f32 v35, v5, s37, v35 +; GENERIC-NEXT: v_fma_f32 v34, v5, s38, v34 +; GENERIC-NEXT: v_fma_f32 v33, v5, s39, v33 +; GENERIC-NEXT: v_fma_f32 v32, v5, s40, v32 +; GENERIC-NEXT: v_fma_f32 v30, v5, s43, v30 +; GENERIC-NEXT: v_fma_f32 v29, v5, s44, v29 +; GENERIC-NEXT: v_fma_f32 v28, v5, s45, v28 +; GENERIC-NEXT: v_fma_f32 v27, v5, s46, v27 +; GENERIC-NEXT: v_fma_f32 v26, v5, s47, v26 +; GENERIC-NEXT: v_fma_f32 v25, v5, s48, v25 +; GENERIC-NEXT: v_fma_f32 v24, v5, s49, v24 +; GENERIC-NEXT: v_fma_f32 v23, v5, s50, v23 +; GENERIC-NEXT: v_fma_f32 v22, v5, s51, v22 +; GENERIC-NEXT: s_waitcnt lgkmcnt(0) +; GENERIC-NEXT: v_fma_f32 v21, v5, s52, v21 +; GENERIC-NEXT: v_fma_f32 v20, v5, s53, v20 +; GENERIC-NEXT: v_fma_f32 v19, v5, s54, v19 +; GENERIC-NEXT: v_fma_f32 v18, v5, s55, v18 +; GENERIC-NEXT: v_fma_f32 v16, v5, s56, v16 +; GENERIC-NEXT: v_fma_f32 v12, v5, s57, v12 +; GENERIC-NEXT: v_fma_f32 v11, v5, s58, v11 +; GENERIC-NEXT: v_fma_f32 v10, v5, s59, v10 +; GENERIC-NEXT: v_fma_f32 v9, v5, s60, v9 +; GENERIC-NEXT: v_fma_f32 v8, v5, s61, v8 +; GENERIC-NEXT: v_fma_f32 v7, v5, s62, v7 +; GENERIC-NEXT: v_fma_f32 v6, v5, s63, v6 +; GENERIC-NEXT: v_fma_f32 v17, v5, s64, v17 +; GENERIC-NEXT: v_fma_f32 v15, v5, s65, v15 +; GENERIC-NEXT: v_fma_f32 v14, v5, s66, v14 +; GENERIC-NEXT: v_fma_f32 v13, v5, s67, v13 +; GENERIC-NEXT: v_fma_f32 v4, v5, s41, v4 +; GENERIC-NEXT: v_fma_f32 v3, v5, s42, v3 +; GENERIC-NEXT: s_cbranch_scc1 .LBB4_1 +; GENERIC-NEXT: ; %bb.2: ; %for.cond.cleanup26 +; GENERIC-NEXT: s_lshl_b32 s0, s8, 8 +; GENERIC-NEXT: s_add_i32 s0, s6, s0 +; GENERIC-NEXT: v_add_i32_e32 v0, vcc, s0, v0 +; GENERIC-NEXT: s_mov_b32 s0, 0x5397829d +; GENERIC-NEXT: v_mul_hi_u32 v0, v0, s0 +; GENERIC-NEXT: s_mov_b32 s0, 0x31000 +; GENERIC-NEXT: v_mov_b32_e32 v1, 0 +; GENERIC-NEXT: v_mov_b32_e32 v2, s3 +; GENERIC-NEXT: v_lshrrev_b32_e32 v0, 4, v0 +; GENERIC-NEXT: v_mul_lo_u32 v0, v0, s0 +; GENERIC-NEXT: s_mov_b32 s7, 0xf000 +; GENERIC-NEXT: s_mov_b32 s6, 0 +; GENERIC-NEXT: s_mov_b64 s[4:5], s[2:3] +; GENERIC-NEXT: v_lshl_b64 v[0:1], v[0:1], 2 +; GENERIC-NEXT: buffer_store_dword v34, v[0:1], s[4:7], 0 addr64 offset:1568 +; GENERIC-NEXT: buffer_store_dword v33, v[0:1], s[4:7], 0 addr64 offset:2352 +; GENERIC-NEXT: v_add_i32_e32 v37, vcc, s2, v0 +; GENERIC-NEXT: v_addc_u32_e32 v38, vcc, v2, v1, vcc +; GENERIC-NEXT: s_movk_i32 s2, 0x310 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_add_i32_e32 v33, vcc, s2, v37 +; GENERIC-NEXT: s_mov_b32 s2, s6 +; GENERIC-NEXT: s_mov_b32 s3, s6 +; GENERIC-NEXT: buffer_store_dword v36, v[0:1], s[4:7], 0 addr64 +; GENERIC-NEXT: buffer_store_dword v35, v[0:1], s[4:7], 0 addr64 offset:784 +; GENERIC-NEXT: v_addc_u32_e32 v34, vcc, 0, v38, vcc +; GENERIC-NEXT: v_mov_b32_e32 v5, s6 +; GENERIC-NEXT: s_waitcnt expcnt(1) +; GENERIC-NEXT: v_mov_b32_e32 v36, s3 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_mov_b32_e32 v35, s2 +; GENERIC-NEXT: s_movk_i32 s2, 0x620 +; GENERIC-NEXT: buffer_store_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:3920 +; GENERIC-NEXT: buffer_store_dwordx2 v[35:36], v[0:1], s[4:7], 0 addr64 offset:840 +; GENERIC-NEXT: buffer_store_dword v32, v[0:1], s[4:7], 0 addr64 offset:3136 +; GENERIC-NEXT: v_add_i32_e32 v0, vcc, s2, v37 +; GENERIC-NEXT: v_addc_u32_e32 v1, vcc, 0, v38, vcc +; GENERIC-NEXT: s_movk_i32 s4, 0x930 +; GENERIC-NEXT: s_mov_b64 s[0:1], 0x310 +; GENERIC-NEXT: s_mov_b64 s[2:3], s[6:7] +; GENERIC-NEXT: s_waitcnt expcnt(2) +; GENERIC-NEXT: v_mov_b32_e32 v4, s6 +; GENERIC-NEXT: v_add_i32_e32 v2, vcc, s4, v37 +; GENERIC-NEXT: buffer_store_dwordx2 v[3:4], v[37:38], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_addc_u32_e32 v3, vcc, 0, v38, vcc +; GENERIC-NEXT: s_movk_i32 s0, 0xc40 +; GENERIC-NEXT: v_add_i32_e32 v4, vcc, s0, v37 +; GENERIC-NEXT: s_mov_b64 s[8:9], 0x620 +; GENERIC-NEXT: s_mov_b64 s[10:11], s[6:7] +; GENERIC-NEXT: v_addc_u32_e32 v5, vcc, 0, v38, vcc +; GENERIC-NEXT: s_movk_i32 s4, 0xf50 +; GENERIC-NEXT: buffer_store_dword v30, v[37:38], s[8:11], 0 addr64 offset:3920 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_add_i32_e32 v30, vcc, s4, v37 +; GENERIC-NEXT: s_mov_b64 s[16:17], 0xc40 +; GENERIC-NEXT: v_addc_u32_e32 v31, vcc, 0, v38, vcc +; GENERIC-NEXT: s_mov_b64 s[18:19], s[6:7] +; GENERIC-NEXT: s_mov_b64 s[12:13], 0x930 +; GENERIC-NEXT: s_mov_b64 s[14:15], s[6:7] +; GENERIC-NEXT: buffer_store_dword v28, v[37:38], s[16:19], 0 addr64 offset:3920 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_add_i32_e32 v28, vcc, s4, v33 +; GENERIC-NEXT: buffer_store_dword v29, v[37:38], s[12:15], 0 addr64 offset:3920 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_addc_u32_e32 v29, vcc, 0, v34, vcc +; GENERIC-NEXT: buffer_store_dwordx2 v[35:36], v[37:38], s[12:15], 0 addr64 offset:3976 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_add_i32_e32 v35, vcc, s4, v0 +; GENERIC-NEXT: s_mov_b64 s[0:1], 0xf50 +; GENERIC-NEXT: v_addc_u32_e32 v36, vcc, 0, v1, vcc +; GENERIC-NEXT: buffer_store_dword v26, v[33:34], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_add_i32_e32 v26, vcc, s4, v2 +; GENERIC-NEXT: buffer_store_dword v27, v[37:38], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_addc_u32_e32 v27, vcc, 0, v3, vcc +; GENERIC-NEXT: v_add_i32_e32 v32, vcc, s4, v4 +; GENERIC-NEXT: v_addc_u32_e32 v33, vcc, 0, v5, vcc +; GENERIC-NEXT: buffer_store_dword v25, v[0:1], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: v_add_i32_e32 v0, vcc, s4, v30 +; GENERIC-NEXT: v_addc_u32_e32 v1, vcc, 0, v31, vcc +; GENERIC-NEXT: buffer_store_dword v24, v[2:3], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: v_add_i32_e32 v2, vcc, s4, v28 +; GENERIC-NEXT: v_addc_u32_e32 v3, vcc, 0, v29, vcc +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_add_i32_e32 v24, vcc, s4, v35 +; GENERIC-NEXT: v_addc_u32_e32 v25, vcc, 0, v36, vcc +; GENERIC-NEXT: buffer_store_dword v23, v[4:5], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: v_add_i32_e32 v4, vcc, s4, v26 +; GENERIC-NEXT: v_addc_u32_e32 v5, vcc, 0, v27, vcc +; GENERIC-NEXT: buffer_store_dword v22, v[30:31], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_add_i32_e32 v22, vcc, s4, v32 +; GENERIC-NEXT: v_addc_u32_e32 v23, vcc, 0, v33, vcc +; GENERIC-NEXT: v_add_i32_e32 v30, vcc, s4, v0 +; GENERIC-NEXT: v_addc_u32_e32 v31, vcc, 0, v1, vcc +; GENERIC-NEXT: buffer_store_dword v21, v[28:29], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: v_add_i32_e32 v28, vcc, s4, v2 +; GENERIC-NEXT: v_addc_u32_e32 v29, vcc, 0, v3, vcc +; GENERIC-NEXT: buffer_store_dword v20, v[35:36], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_add_i32_e32 v20, vcc, s4, v24 +; GENERIC-NEXT: v_addc_u32_e32 v21, vcc, 0, v25, vcc +; GENERIC-NEXT: v_add_i32_e32 v34, vcc, s4, v4 +; GENERIC-NEXT: v_addc_u32_e32 v35, vcc, 0, v5, vcc +; GENERIC-NEXT: buffer_store_dword v19, v[26:27], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: v_add_i32_e32 v26, vcc, s4, v22 +; GENERIC-NEXT: v_addc_u32_e32 v27, vcc, 0, v23, vcc +; GENERIC-NEXT: buffer_store_dword v18, v[32:33], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: s_waitcnt expcnt(0) +; GENERIC-NEXT: v_add_i32_e32 v18, vcc, s4, v30 +; GENERIC-NEXT: v_addc_u32_e32 v19, vcc, 0, v31, vcc +; GENERIC-NEXT: buffer_store_dword v16, v[0:1], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: v_add_i32_e32 v0, vcc, 0xf50, v28 +; GENERIC-NEXT: v_addc_u32_e32 v1, vcc, 0, v29, vcc +; GENERIC-NEXT: buffer_store_dword v12, v[2:3], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: buffer_store_dword v11, v[24:25], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: buffer_store_dword v10, v[4:5], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: buffer_store_dword v9, v[22:23], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: buffer_store_dword v8, v[30:31], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: buffer_store_dword v7, v[28:29], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: buffer_store_dword v6, v[20:21], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: buffer_store_dword v17, v[34:35], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: buffer_store_dword v15, v[26:27], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: buffer_store_dword v14, v[18:19], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: buffer_store_dword v13, v[0:1], s[0:3], 0 addr64 offset:3920 +; GENERIC-NEXT: s_endpgm +; +; GENERIC-GCNTRACKERS-LABEL: excess_soft_clause_reg_pressure: +; GENERIC-GCNTRACKERS: ; %bb.0: ; %entry +; GENERIC-GCNTRACKERS-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9 +; GENERIC-GCNTRACKERS-NEXT: v_lshlrev_b32_e32 v1, 2, v0 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s30, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v3, 0 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s31, 0xf000 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt lgkmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s5 +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v1, vcc, s4, v1 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc +; GENERIC-GCNTRACKERS-NEXT: s_movk_i32 s4, 0x1000 +; GENERIC-GCNTRACKERS-NEXT: s_movk_i32 s5, 0x2000 +; GENERIC-GCNTRACKERS-NEXT: s_movk_i32 s7, 0x3000 +; GENERIC-GCNTRACKERS-NEXT: s_movk_i32 s9, 0x1040 +; GENERIC-GCNTRACKERS-NEXT: s_movk_i32 s10, 0x2040 +; GENERIC-GCNTRACKERS-NEXT: s_movk_i32 s11, 0x3040 +; GENERIC-GCNTRACKERS-NEXT: s_movk_i32 s33, 0x310 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v6, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v7, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v8, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v9, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v10, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v11, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v12, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v16, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v18, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v19, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v20, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v21, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v22, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v23, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v24, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v25, 0 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s34, s30 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v26, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v27, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v28, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v29, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v30, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v31, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v33, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v34, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v35, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v37, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v17, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v15, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v14, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v13, 0 +; GENERIC-GCNTRACKERS-NEXT: .LBB4_1: ; %for.cond28.preheader +; GENERIC-GCNTRACKERS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s28, s30 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s29, s30 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v38, v[1:2], s[28:31], 0 addr64 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v36, v[1:2], s[28:31], 0 addr64 offset:196 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v32, v[1:2], s[28:31], 0 addr64 offset:392 +; GENERIC-GCNTRACKERS-NEXT: buffer_load_dword v5, v[1:2], s[28:31], 0 addr64 offset:588 +; GENERIC-GCNTRACKERS-NEXT: s_load_dwordx16 s[36:51], s[0:1], 0x0 +; GENERIC-GCNTRACKERS-NEXT: s_load_dwordx16 s[12:27], s[0:1], 0x10 +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v1, vcc, s33, v1 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(3) lgkmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v37, v38, s36, v37 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v35, v38, s37, v35 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v34, v38, s38, v34 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v33, v38, s39, v33 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v31, v38, s40, v31 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v4, v38, s41, v4 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v3, v38, s42, v3 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v30, v38, s43, v30 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v29, v38, s44, v29 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v28, v38, s45, v28 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v27, v38, s46, v27 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v26, v38, s47, v26 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v25, v38, s48, v25 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v24, v38, s49, v24 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v23, v38, s50, v23 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v22, v38, s51, v22 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v21, v38, s12, v21 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v20, v38, s13, v20 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v19, v38, s14, v19 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v18, v38, s15, v18 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v16, v38, s16, v16 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v12, v38, s17, v12 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v11, v38, s18, v11 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v10, v38, s19, v10 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v9, v38, s20, v9 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v8, v38, s21, v8 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v7, v38, s22, v7 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v6, v38, s23, v6 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v17, v38, s24, v17 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v15, v38, s25, v15 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v14, v38, s26, v14 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v13, v38, s27, v13 +; GENERIC-GCNTRACKERS-NEXT: s_load_dwordx16 s[36:51], s[0:1], s4 +; GENERIC-GCNTRACKERS-NEXT: s_load_dwordx16 s[12:27], s[0:1], s9 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(2) lgkmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v37, v36, s36, v37 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v35, v36, s37, v35 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v34, v36, s38, v34 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v33, v36, s39, v33 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v31, v36, s40, v31 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v4, v36, s41, v4 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v3, v36, s42, v3 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v30, v36, s43, v30 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v29, v36, s44, v29 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v28, v36, s45, v28 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v27, v36, s46, v27 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v26, v36, s47, v26 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v25, v36, s48, v25 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v24, v36, s49, v24 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v23, v36, s50, v23 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v22, v36, s51, v22 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v21, v36, s12, v21 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v20, v36, s13, v20 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v19, v36, s14, v19 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v18, v36, s15, v18 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v16, v36, s16, v16 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v12, v36, s17, v12 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v11, v36, s18, v11 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v10, v36, s19, v10 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v9, v36, s20, v9 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v8, v36, s21, v8 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v7, v36, s22, v7 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v6, v36, s23, v6 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v17, v36, s24, v17 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v15, v36, s25, v15 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v14, v36, s26, v14 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v13, v36, s27, v13 +; GENERIC-GCNTRACKERS-NEXT: s_load_dwordx16 s[36:51], s[0:1], s5 +; GENERIC-GCNTRACKERS-NEXT: s_load_dwordx16 s[12:27], s[0:1], s10 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(1) lgkmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v36, v32, s36, v37 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v35, v32, s37, v35 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v34, v32, s38, v34 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v33, v32, s39, v33 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v31, v32, s40, v31 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v4, v32, s41, v4 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v3, v32, s42, v3 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v30, v32, s43, v30 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v29, v32, s44, v29 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v28, v32, s45, v28 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v27, v32, s46, v27 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v26, v32, s47, v26 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v25, v32, s48, v25 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v24, v32, s49, v24 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v23, v32, s50, v23 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v22, v32, s51, v22 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v21, v32, s12, v21 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v20, v32, s13, v20 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v19, v32, s14, v19 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v18, v32, s15, v18 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v16, v32, s16, v16 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v12, v32, s17, v12 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v11, v32, s18, v11 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v10, v32, s19, v10 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v9, v32, s20, v9 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v8, v32, s21, v8 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v7, v32, s22, v7 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v6, v32, s23, v6 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v17, v32, s24, v17 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v15, v32, s25, v15 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v14, v32, s26, v14 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v13, v32, s27, v13 +; GENERIC-GCNTRACKERS-NEXT: s_load_dwordx16 s[36:51], s[0:1], s7 +; GENERIC-GCNTRACKERS-NEXT: s_load_dwordx16 s[12:27], s[0:1], s11 +; GENERIC-GCNTRACKERS-NEXT: s_add_u32 s0, s0, 0x4000 +; GENERIC-GCNTRACKERS-NEXT: s_addc_u32 s1, s1, 0 +; GENERIC-GCNTRACKERS-NEXT: s_add_i32 s34, s34, 1 +; GENERIC-GCNTRACKERS-NEXT: s_cmpk_lg_i32 s34, 0x200 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v37, v5, s36, v36 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v35, v5, s37, v35 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v34, v5, s38, v34 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v33, v5, s39, v33 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v31, v5, s40, v31 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v4, v5, s41, v4 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v3, v5, s42, v3 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v30, v5, s43, v30 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v29, v5, s44, v29 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v28, v5, s45, v28 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v27, v5, s46, v27 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v26, v5, s47, v26 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v25, v5, s48, v25 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v24, v5, s49, v24 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v23, v5, s50, v23 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v22, v5, s51, v22 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v21, v5, s12, v21 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v20, v5, s13, v20 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v19, v5, s14, v19 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v18, v5, s15, v18 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v16, v5, s16, v16 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v12, v5, s17, v12 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v11, v5, s18, v11 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v10, v5, s19, v10 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v9, v5, s20, v9 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v8, v5, s21, v8 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v7, v5, s22, v7 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v6, v5, s23, v6 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v17, v5, s24, v17 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v15, v5, s25, v15 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v14, v5, s26, v14 +; GENERIC-GCNTRACKERS-NEXT: v_fma_f32 v13, v5, s27, v13 +; GENERIC-GCNTRACKERS-NEXT: s_cbranch_scc1 .LBB4_1 +; GENERIC-GCNTRACKERS-NEXT: ; %bb.2: ; %for.cond.cleanup26 +; GENERIC-GCNTRACKERS-NEXT: s_lshl_b32 s0, s8, 8 +; GENERIC-GCNTRACKERS-NEXT: s_add_i32 s0, s6, s0 +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v0, vcc, s0, v0 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s0, 0x5397829d +; GENERIC-GCNTRACKERS-NEXT: v_mul_hi_u32 v0, v0, s0 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s0, 0x31000 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v1, 0 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v2, s3 +; GENERIC-GCNTRACKERS-NEXT: v_lshrrev_b32_e32 v0, 4, v0 +; GENERIC-GCNTRACKERS-NEXT: v_mul_lo_u32 v0, v0, s0 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s7, 0xf000 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s6, 0 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b64 s[4:5], s[2:3] +; GENERIC-GCNTRACKERS-NEXT: v_lshl_b64 v[0:1], v[0:1], 2 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s3, s6 +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v38, vcc, s2, v0 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v39, vcc, v2, v1, vcc +; GENERIC-GCNTRACKERS-NEXT: s_movk_i32 s2, 0x310 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v35, v[0:1], s[4:7], 0 addr64 offset:784 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v35, vcc, s2, v38 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b32 s2, s6 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v36, vcc, 0, v39, vcc +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v34, v[0:1], s[4:7], 0 addr64 offset:1568 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v33, v[0:1], s[4:7], 0 addr64 offset:2352 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v33, s3 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v32, s2 +; GENERIC-GCNTRACKERS-NEXT: s_movk_i32 s2, 0x620 +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v5, s6 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v37, v[0:1], s[4:7], 0 addr64 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx2 v[32:33], v[0:1], s[4:7], 0 addr64 offset:840 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v31, v[0:1], s[4:7], 0 addr64 offset:3136 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v0, vcc, s2, v38 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v1, vcc, 0, v39, vcc +; GENERIC-GCNTRACKERS-NEXT: s_movk_i32 s4, 0x930 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b64 s[0:1], 0x310 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b64 s[2:3], s[6:7] +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_mov_b32_e32 v4, s6 +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v2, vcc, s4, v38 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx2 v[3:4], v[38:39], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v39, vcc +; GENERIC-GCNTRACKERS-NEXT: s_movk_i32 s0, 0xc40 +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v4, vcc, s0, v38 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b64 s[8:9], 0x620 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b64 s[10:11], s[6:7] +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v5, vcc, 0, v39, vcc +; GENERIC-GCNTRACKERS-NEXT: s_movk_i32 s4, 0xf50 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v30, v[38:39], s[8:11], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v30, vcc, s4, v38 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b64 s[16:17], 0xc40 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v31, vcc, 0, v39, vcc +; GENERIC-GCNTRACKERS-NEXT: s_mov_b64 s[18:19], s[6:7] +; GENERIC-GCNTRACKERS-NEXT: s_mov_b64 s[12:13], 0x930 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b64 s[14:15], s[6:7] +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v28, v[38:39], s[16:19], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v28, vcc, s4, v35 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v29, v[38:39], s[12:15], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v29, vcc, 0, v36, vcc +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dwordx2 v[32:33], v[38:39], s[12:15], 0 addr64 offset:3976 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v32, vcc, s4, v0 +; GENERIC-GCNTRACKERS-NEXT: s_mov_b64 s[0:1], 0xf50 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v33, vcc, 0, v1, vcc +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v26, v[35:36], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v26, vcc, s4, v2 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v27, v[38:39], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v27, vcc, 0, v3, vcc +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v34, vcc, s4, v4 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v35, vcc, 0, v5, vcc +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v25, v[0:1], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v0, vcc, s4, v30 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v1, vcc, 0, v31, vcc +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v24, v[2:3], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v2, vcc, s4, v28 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v3, vcc, 0, v29, vcc +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v24, vcc, s4, v32 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v25, vcc, 0, v33, vcc +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v23, v[4:5], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v4, vcc, s4, v26 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v5, vcc, 0, v27, vcc +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v22, v[30:31], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v22, vcc, s4, v34 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v23, vcc, 0, v35, vcc +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v30, vcc, s4, v0 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v31, vcc, 0, v1, vcc +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v21, v[28:29], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v28, vcc, s4, v2 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v29, vcc, 0, v3, vcc +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v20, v[32:33], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v20, vcc, s4, v24 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v21, vcc, 0, v25, vcc +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v32, vcc, s4, v4 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v33, vcc, 0, v5, vcc +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v19, v[26:27], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v26, vcc, s4, v22 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v27, vcc, 0, v23, vcc +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v18, v[34:35], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: s_waitcnt expcnt(0) +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v18, vcc, s4, v30 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v19, vcc, 0, v31, vcc +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v16, v[0:1], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: v_add_i32_e32 v0, vcc, 0xf50, v28 +; GENERIC-GCNTRACKERS-NEXT: v_addc_u32_e32 v1, vcc, 0, v29, vcc +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v12, v[2:3], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v11, v[24:25], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v10, v[4:5], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v9, v[22:23], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v8, v[30:31], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v7, v[28:29], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v6, v[20:21], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v17, v[32:33], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v15, v[26:27], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v14, v[18:19], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: buffer_store_dword v13, v[0:1], s[0:3], 0 addr64 offset:3920 +; GENERIC-GCNTRACKERS-NEXT: s_endpgm entry: %i = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() %i2 = load i64, ptr addrspace(4) %i, align 8
diff --git a/llvm/test/CodeGen/AMDGPU/scmp.ll b/llvm/test/CodeGen/AMDGPU/scmp.ll index 62aa991..c23139a 100644 --- a/llvm/test/CodeGen/AMDGPU/scmp.ll +++ b/llvm/test/CodeGen/AMDGPU/scmp.ll
@@ -49,28 +49,20 @@ ; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[2:3], v[6:7] -; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo -; GFX12-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[4:5] -; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo -; GFX12-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[2:3], v[6:7] -; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo +; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s0, v[0:1], v[4:5] +; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s1, v[2:3], v[6:7] ; GFX12-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5] -; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.l, v8.l -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3) -; GFX12-SDAG-NEXT: v_mov_b16_e32 v1.l, v10.l +; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s2, v[2:3], v[6:7] +; GFX12-SDAG-NEXT: v_cmp_eq_u64_e64 s3, v[2:3], v[6:7] +; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX12-SDAG-NEXT: v_cndmask_b16 v0.h, 0, 1, s0 +; GFX12-SDAG-NEXT: v_cndmask_b16 v1.l, 0, 1, s1 ; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo -; GFX12-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[6:7] -; GFX12-SDAG-NEXT: v_mov_b16_e32 v2.l, v9.l -; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.l, v4.l -; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-NEXT: v_cndmask_b16 v0.h, v1.l, v2.l, vcc_lo -; GFX12-SDAG-NEXT: v_cndmask_b16 v1.l, v0.l, v3.l, vcc_lo +; GFX12-SDAG-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GFX12-SDAG-NEXT: v_cndmask_b16 v1.h, 0, 1, s2 +; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-SDAG-NEXT: v_cndmask_b16 v0.h, v1.l, v0.h, s3 +; GFX12-SDAG-NEXT: v_cndmask_b16 v1.l, v1.h, v0.l, s3 ; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX12-SDAG-NEXT: v_and_b16 v0.l, 1, v0.h ; GFX12-SDAG-NEXT: v_and_b32_e32 v1, 1, v1 @@ -315,8 +307,8 @@ ; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX12-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8 +; GFX12-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2) ; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v0.l, v1.l ; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) @@ -507,42 +499,37 @@ ; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX12-SDAG-NEXT: v_bfe_i32 v4, v4, 0, 8 +; GFX12-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX12-SDAG-NEXT: v_bfe_i32 v5, v5, 0, 8 -; GFX12-SDAG-NEXT: v_bfe_i32 v8, v3, 0, 8 -; GFX12-SDAG-NEXT: v_bfe_i32 v6, v6, 0, 8 ; GFX12-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8 +; GFX12-SDAG-NEXT: v_bfe_i32 v6, v6, 0, 8 +; GFX12-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8 ; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v0.l, v4.l -; GFX12-SDAG-NEXT: v_bfe_i32 v9, v2, 0, 8 -; GFX12-SDAG-NEXT: v_mov_b16_e32 v2.l, v5.l ; GFX12-SDAG-NEXT: v_bfe_i32 v7, v7, 0, 8 -; GFX12-SDAG-NEXT: v_mov_b16_e32 v5.l, v8.l -; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo -; GFX12-SDAG-NEXT: v_cmp_ge_i16_e32 vcc_lo, v0.l, v4.l -; GFX12-SDAG-NEXT: v_mov_b16_e32 v4.l, v6.l -; GFX12-SDAG-NEXT: v_mov_b16_e32 v6.l, v7.l -; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-SDAG-NEXT: v_cndmask_b32_e32 v0, -1, v3, vcc_lo -; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.l, v9.l -; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v1.l, v2.l -; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v3.l, v4.l +; GFX12-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 8 ; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo -; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v5.l, v6.l +; GFX12-SDAG-NEXT: v_cmp_ge_i16_e32 vcc_lo, v0.l, v4.l +; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-SDAG-NEXT: v_cndmask_b32_e32 v0, -1, v8, vcc_lo +; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v1.l, v5.l +; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo +; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v2.l, v6.l +; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo +; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v3.l, v7.l ; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo -; GFX12-SDAG-NEXT: v_cmp_ge_i16_e32 vcc_lo, v1.l, v2.l +; GFX12-SDAG-NEXT: v_cmp_ge_i16_e32 vcc_lo, v1.l, v5.l ; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-SDAG-NEXT: v_cndmask_b32_e32 v1, -1, v7, vcc_lo -; GFX12-SDAG-NEXT: v_cmp_ge_i16_e32 vcc_lo, v3.l, v4.l +; GFX12-SDAG-NEXT: v_cndmask_b32_e32 v1, -1, v4, vcc_lo +; GFX12-SDAG-NEXT: v_cmp_ge_i16_e32 vcc_lo, v2.l, v6.l ; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-SDAG-NEXT: v_cndmask_b32_e32 v2, -1, v8, vcc_lo -; GFX12-SDAG-NEXT: v_cmp_ge_i16_e32 vcc_lo, v5.l, v6.l +; GFX12-SDAG-NEXT: v_cmp_ge_i16_e32 vcc_lo, v3.l, v7.l ; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) ; GFX12-SDAG-NEXT: v_cndmask_b32_e32 v3, -1, v9, vcc_lo ; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/sext-in-reg-vector-shuffle.ll b/llvm/test/CodeGen/AMDGPU/sext-in-reg-vector-shuffle.ll index 62bfc2d..59d4dc2 100644 --- a/llvm/test/CodeGen/AMDGPU/sext-in-reg-vector-shuffle.ll +++ b/llvm/test/CodeGen/AMDGPU/sext-in-reg-vector-shuffle.ll
@@ -8,31 +8,29 @@ ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, 0 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0 ; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v0, s[2:3] ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2 -; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v6, 24, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1 +; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v5, 24, v2 +; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v7, 24, v1 +; GFX11-TRUE16-NEXT: v_bfe_i32 v8, v2, 0, 8 ; GFX11-TRUE16-NEXT: v_ashrrev_i16 v0.l, 8, v1.l ; GFX11-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 -; GFX11-TRUE16-NEXT: v_bfe_i32 v10, v4, 0, 8 -; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v9, 24, v2 -; GFX11-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 8 ; GFX11-TRUE16-NEXT: v_ashrrev_i16 v0.h, 8, v2.l -; GFX11-TRUE16-NEXT: v_bfe_i32 v7, v3, 0, 8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l -; GFX11-TRUE16-NEXT: v_cvt_f16_i16_e32 v2.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l +; GFX11-TRUE16-NEXT: v_bfe_i32 v9, v3, 0, 8 +; GFX11-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8 ; GFX11-TRUE16-NEXT: v_cvt_f16_i16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_cvt_f16_i16_e32 v3.h, v1.l ; GFX11-TRUE16-NEXT: v_cvt_f16_i16_e32 v1.l, v0.h -; GFX11-TRUE16-NEXT: v_cvt_f16_i16_e32 v0.l, v9.l -; GFX11-TRUE16-NEXT: v_cvt_f16_i16_e32 v3.h, v4.l -; GFX11-TRUE16-NEXT: v_cvt_f16_i16_e32 v1.h, v5.l +; GFX11-TRUE16-NEXT: v_cvt_f16_i16_e32 v1.h, v8.l +; GFX11-TRUE16-NEXT: v_cvt_f16_i16_e32 v2.l, v7.l +; GFX11-TRUE16-NEXT: v_cvt_f16_i16_e32 v0.l, v5.l ; GFX11-TRUE16-NEXT: v_cvt_f16_i16_e32 v2.h, v6.l -; GFX11-TRUE16-NEXT: v_cvt_f16_i16_e32 v0.h, v7.l -; GFX11-TRUE16-NEXT: global_store_b128 v8, v[0:3], s[0:1] +; GFX11-TRUE16-NEXT: v_cvt_f16_i16_e32 v0.h, v9.l +; GFX11-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1] ; GFX11-TRUE16-NEXT: s_endpgm ; ; GFX11-FAKE16-LABEL: v_sext_in_reg_i8_i16_shuffle_vector:
diff --git a/llvm/test/CodeGen/AMDGPU/ucmp.ll b/llvm/test/CodeGen/AMDGPU/ucmp.ll index 2eef0cd..abd1896 100644 --- a/llvm/test/CodeGen/AMDGPU/ucmp.ll +++ b/llvm/test/CodeGen/AMDGPU/ucmp.ll
@@ -49,28 +49,20 @@ ; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[2:3], v[6:7] -; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo -; GFX12-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[4:5] -; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo -; GFX12-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[2:3], v[6:7] -; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo +; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s0, v[0:1], v[4:5] +; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s1, v[2:3], v[6:7] ; GFX12-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5] -; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.l, v8.l -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3) -; GFX12-SDAG-NEXT: v_mov_b16_e32 v1.l, v10.l +; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s2, v[2:3], v[6:7] +; GFX12-SDAG-NEXT: v_cmp_eq_u64_e64 s3, v[2:3], v[6:7] +; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0) +; GFX12-SDAG-NEXT: v_cndmask_b16 v0.h, 0, 1, s0 +; GFX12-SDAG-NEXT: v_cndmask_b16 v1.l, 0, 1, s1 ; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo -; GFX12-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[6:7] -; GFX12-SDAG-NEXT: v_mov_b16_e32 v2.l, v9.l -; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.l, v4.l -; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0) -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-NEXT: v_cndmask_b16 v0.h, v1.l, v2.l, vcc_lo -; GFX12-SDAG-NEXT: v_cndmask_b16 v1.l, v0.l, v3.l, vcc_lo +; GFX12-SDAG-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo +; GFX12-SDAG-NEXT: v_cndmask_b16 v1.h, 0, 1, s2 +; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-SDAG-NEXT: v_cndmask_b16 v0.h, v1.l, v0.h, s3 +; GFX12-SDAG-NEXT: v_cndmask_b16 v1.l, v1.h, v0.l, s3 ; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX12-SDAG-NEXT: v_and_b16 v0.l, 1, v0.h ; GFX12-SDAG-NEXT: v_and_b32_e32 v1, 1, v1
diff --git a/llvm/test/CodeGen/AMDGPU/urem-constant-i128.ll b/llvm/test/CodeGen/AMDGPU/urem-constant-i128.ll index f5fe60b..76f7c97 100644 --- a/llvm/test/CodeGen/AMDGPU/urem-constant-i128.ll +++ b/llvm/test/CodeGen/AMDGPU/urem-constant-i128.ll
@@ -1,48 +1,39 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 ; RUN: llc -global-isel=0 -mtriple=amdgpu6.00 < %s | FileCheck %s --implicit-check-not=s_swappc_b64 ; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 < %s | FileCheck %s --implicit-check-not=s_swappc_b64 ; Check that AMDGPU legalizes constant i128 UREM without emitting a libcall. define i128 @urem_i128_65(i128 %src) { -; CHECK-LABEL: urem_i128_65: -; CHECK: s_setpc_b64 %result = urem i128 %src, 65 ret i128 %result } define i128 @urem_i128_65_optsize(i128 %src) #0 { -; CHECK-LABEL: urem_i128_65_optsize: -; CHECK: s_setpc_b64 %result = urem i128 %src, 65 ret i128 %result } define i128 @urem_i128_66(i128 %src) { -; CHECK-LABEL: urem_i128_66: -; CHECK: s_setpc_b64 %result = urem i128 %src, 66 ret i128 %result } define i128 @urem_i128_67(i128 %src) { -; CHECK-LABEL: urem_i128_67: -; CHECK: s_setpc_b64 %result = urem i128 %src, 67 ret i128 %result } define i128 @urem_i128_68(i128 %src) { -; CHECK-LABEL: urem_i128_68: -; CHECK: s_setpc_b64 %result = urem i128 %src, 68 ret i128 %result } define i128 @urem_i128_127(i128 %src) { -; CHECK-LABEL: urem_i128_127: -; CHECK: s_setpc_b64 %result = urem i128 %src, 127 ret i128 %result } attributes #0 = { optsize } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll index bb95fce..d4dcece 100644 --- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll +++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
@@ -65,8 +65,8 @@ ; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v2i8: ; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -96,8 +96,8 @@ ; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l ; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -208,8 +208,8 @@ ; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v3i8: ; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v2.l @@ -244,8 +244,8 @@ ; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v2.l @@ -429,8 +429,8 @@ ; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v4i8: ; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l @@ -438,11 +438,9 @@ ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v1, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.l, v2.l +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v2.l, v1.l ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_smax_v4i8: @@ -497,8 +495,8 @@ ; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l @@ -506,11 +504,9 @@ ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v1, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.l, v2.l +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v2.l, v1.l ; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_smax_v4i8: @@ -760,33 +756,23 @@ ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v3, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l -; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v5.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v5.l +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v4, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v6, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v3.l, v7.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v0, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.h -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v6, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.l, v2.l +; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v3.l, v1.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4 +; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v2.l, v5.l ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -859,33 +845,23 @@ ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v3, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l -; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v5.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v5.l +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v4, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v6, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v3.l, v7.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v0, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.h -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v6, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.l, v2.l +; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v3.l, v1.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4 +; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v2.l, v5.l ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l ; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -1235,52 +1211,39 @@ ; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v16i8: ; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v16, v4, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v5, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v9, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v8, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v11, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v7, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v15, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v3, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v11, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v15, v15, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v9.l +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v12, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l -; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v17.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v12, v12, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v1.h, v7.l, v8.l -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v0, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v4.l, v5.l -; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v1.l, v3.l +; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v7.l, v15.l +; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v1.h, v3.l, v11.l +; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v5.l, v13.l +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v8, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v10, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l -; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.h, v1.h, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v14, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v0, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.h, v1.h, v1.l +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v14, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v5.l, v4.l +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.l ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.l -; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v2.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l -; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6 -; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v1.l, v4.l -; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v5.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v2.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v6, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.h, v0.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v6.l, v1.l +; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v2.l, v7.l +; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v3.l, v9.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v4 +; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.h, v1.l, v0.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v2, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -1378,52 +1341,39 @@ ; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v16, v4, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v5, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v9, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v8, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v11, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v7, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v15, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v3, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v11, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v15, v15, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v9.l +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v12, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l -; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v17.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v12, v12, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v1.h, v7.l, v8.l -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v0, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v4.l, v5.l -; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v1.l, v3.l +; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v7.l, v15.l +; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v1.h, v3.l, v11.l +; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v5.l, v13.l +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v8, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v10, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l -; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.h, v1.h, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v14, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v0, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.h, v1.h, v1.l +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v14, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v5.l, v4.l +; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.l ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.l -; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v2.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l -; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6 -; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v1.l, v4.l -; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v5.l, v0.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v2.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v6, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.h, v0.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v6.l, v1.l +; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v2.l, v7.l +; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v3.l, v9.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v4 +; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.h, v1.l, v0.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v2, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l ; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ;
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll index c51bda6f..950754d 100644 --- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll +++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
@@ -65,8 +65,8 @@ ; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v2i8: ; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -96,8 +96,8 @@ ; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l ; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -208,8 +208,8 @@ ; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v3i8: ; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v2.l @@ -244,8 +244,8 @@ ; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v2.l @@ -429,8 +429,8 @@ ; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v4i8: ; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l @@ -438,11 +438,9 @@ ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v1, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.l, v2.l +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v2.l, v1.l ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_smin_v4i8: @@ -497,8 +495,8 @@ ; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l @@ -506,11 +504,9 @@ ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v1, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.l, v2.l +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v2.l, v1.l ; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_smin_v4i8: @@ -760,33 +756,23 @@ ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v3, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l -; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v5.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v5.l +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v4, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v6, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v3.l, v7.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v0, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.h -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v6, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.l, v2.l +; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v3.l, v1.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4 +; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v2.l, v5.l ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -859,33 +845,23 @@ ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v3, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l -; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v5.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v5.l +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v4, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v6, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v3.l, v7.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v0, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.h -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v6, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.l, v2.l +; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v3.l, v1.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4 +; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v2.l, v5.l ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l ; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -1235,52 +1211,39 @@ ; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v16i8: ; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v16, v4, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v5, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v9, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v8, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v11, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v7, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v15, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v3, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v11, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v15, v15, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v9.l +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v12, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l -; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v17.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v12, v12, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v1.h, v7.l, v8.l -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v0, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v4.l, v5.l -; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v1.l, v3.l +; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v7.l, v15.l +; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v1.h, v3.l, v11.l +; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v5.l, v13.l +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v8, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v10, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l -; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.h, v1.h, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v14, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v0, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.h, v1.h, v1.l +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v14, 0, 8 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v5.l, v4.l +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.l ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.l -; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v2.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l -; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6 -; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v1.l, v4.l -; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v5.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v2.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v6, 0, 8 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.h, v0.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v6.l, v1.l +; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v2.l, v7.l +; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v3.l, v9.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v4 +; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.h, v1.l, v0.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v2, 0, 8 ; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -1378,52 +1341,39 @@ ; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v16, v4, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v5, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v9, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v8, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v11, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v7, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v15, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v3, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v11, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v15, v15, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v9.l +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v12, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l -; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v17.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v12, v12, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v1.h, v7.l, v8.l -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v0, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v4.l, v5.l -; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v1.l, v3.l +; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v7.l, v15.l +; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v1.h, v3.l, v11.l +; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v5.l, v13.l +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v8, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v10, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l -; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.h, v1.h, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v14, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v0, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.h, v1.h, v1.l +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v14, 0, 8 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v5.l, v4.l +; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.l ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.l -; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v2.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l -; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6 -; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v1.l, v4.l -; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v5.l, v0.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v2.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v6, 0, 8 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.h, v0.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v6.l, v1.l +; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v2.l, v7.l +; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v3.l, v9.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v4 +; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.h, v1.l, v0.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v2, 0, 8 ; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l ; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ;