src/amd/compiler/aco_insert_fp_mode.cpp - third_party/mesa - Git at Google

 /*
  * Copyright © 2025 Valve Corporation
  *
  * SPDX-License-Identifier: MIT
  */

 #include "aco_builder.h"
 #include "aco_ir.h"

 #include <vector>

 namespace aco {

 namespace {

 enum mode_field : uint8_t {
    mode_round32 = 0,
    mode_round16_64,
    mode_denorm32,
    mode_denorm16_64,
    mode_fp16_ovfl,

    mode_field_count,
 };

 using mode_mask = uint8_t;
 static_assert(mode_field_count <= sizeof(mode_mask) * 8, "larger mode_mask needed");

 struct fp_mode_state {
    uint8_t fields[mode_field_count] = {};
    mode_mask dirty = 0; /* BITFIELD_BIT(enum mode_field) */

    fp_mode_state() = default;

    fp_mode_state(float_mode mode)
    {
       fields[mode_round32] = mode.round32;
       fields[mode_round16_64] = mode.round16_64;
       fields[mode_denorm32] = mode.denorm32;
       fields[mode_denorm16_64] = mode.denorm16_64;
       fields[mode_fp16_ovfl] = 0;
    }

    void join(const fp_mode_state& other)
    {
       dirty |= other.dirty;
       for (unsigned i = 0; i < mode_field_count; i++) {
          if (fields[i] != other.fields[i])
             dirty |= BITFIELD_BIT(i);
       }
    }

    bool require(mode_field field, uint8_t val)
    {
       if (fields[field] == val && !(dirty & BITFIELD_BIT(field)))
          return false;

       fields[field] = val;
       dirty |= BITFIELD_BIT(field);
       return true;
    }

    uint8_t round() const { return fields[mode_round32] | (fields[mode_round16_64] << 2); }

    uint8_t denorm() const { return fields[mode_denorm32] | (fields[mode_denorm16_64] << 2); }
 };

 struct fp_mode_ctx {
    std::vector<fp_mode_state> block_states;
    Program* program;
 };

 void
 emit_set_mode(Builder& bld, const fp_mode_state& state)
 {
    bool set_round = state.dirty & (BITFIELD_BIT(mode_round32) | BITFIELD_BIT(mode_round16_64));
    bool set_denorm = state.dirty & (BITFIELD_BIT(mode_denorm32) | BITFIELD_BIT(mode_denorm16_64));
    bool set_fp16_ovfl = state.dirty & BITFIELD_BIT(mode_fp16_ovfl);

    if (bld.program->gfx_level >= GFX10) {
       if (set_round)
          bld.sopp(aco_opcode::s_round_mode, state.round());
       if (set_denorm)
          bld.sopp(aco_opcode::s_denorm_mode, state.denorm());
    } else if (set_round || set_denorm) {
       /* "((size - 1) << 11) | register" (MODE is encoded as register 1) */
       uint8_t val = state.round() | (state.denorm() << 4);
       bld.sopk(aco_opcode::s_setreg_imm32_b32, Operand::literal32(val), (7 << 11) | 1);
    }

    if (set_fp16_ovfl) {
       /* "((size - 1) << 11 | (offset << 6) | register" (MODE is encoded as register 1, we
        * want to set a single bit at offset 23)
        */
       bld.sopk(aco_opcode::s_setreg_imm32_b32, Operand::literal32(state.fields[mode_fp16_ovfl]),
                (0 << 11) | (23 << 6) | 1);
    }
 }

 mode_mask
 vmem_default_needs(Instruction* instr)
 {
    switch (instr->opcode) {
    case aco_opcode::buffer_atomic_fcmpswap:
    case aco_opcode::buffer_atomic_fmin:
    case aco_opcode::buffer_atomic_fmax:
    case aco_opcode::buffer_atomic_add_f32:
    case aco_opcode::flat_atomic_fcmpswap:
    case aco_opcode::flat_atomic_fmin:
    case aco_opcode::flat_atomic_fmax:
    case aco_opcode::flat_atomic_add_f32:
    case aco_opcode::global_atomic_fcmpswap:
    case aco_opcode::global_atomic_fmin:
    case aco_opcode::global_atomic_fmax:
    case aco_opcode::global_atomic_add_f32:
    case aco_opcode::image_atomic_fcmpswap:
    case aco_opcode::image_atomic_fmin:
    case aco_opcode::image_atomic_fmax:
    case aco_opcode::image_atomic_add_flt: return BITFIELD_BIT(mode_denorm32);
    case aco_opcode::buffer_atomic_fcmpswap_x2:
    case aco_opcode::buffer_atomic_fmin_x2:
    case aco_opcode::buffer_atomic_fmax_x2:
    case aco_opcode::buffer_atomic_pk_add_f16:
    case aco_opcode::buffer_atomic_pk_add_bf16:
    case aco_opcode::flat_atomic_fcmpswap_x2:
    case aco_opcode::flat_atomic_fmin_x2:
    case aco_opcode::flat_atomic_fmax_x2:
    case aco_opcode::flat_atomic_pk_add_f16:
    case aco_opcode::flat_atomic_pk_add_bf16:
    case aco_opcode::global_atomic_fcmpswap_x2:
    case aco_opcode::global_atomic_fmin_x2:
    case aco_opcode::global_atomic_fmax_x2:
    case aco_opcode::global_atomic_pk_add_f16:
    case aco_opcode::global_atomic_pk_add_bf16:
    case aco_opcode::image_atomic_pk_add_f16:
    case aco_opcode::image_atomic_pk_add_bf16: return BITFIELD_BIT(mode_denorm16_64);
    default: return 0;
    }
 }

 mode_mask
 instr_default_needs(fp_mode_ctx* ctx, Block* block, Instruction* instr)
 {
    if ((instr->isVMEM() || instr->isFlatLike()) && ctx->program->gfx_level < GFX12)
       return vmem_default_needs(instr);

    switch (instr->opcode) {
    case aco_opcode::s_branch:
    case aco_opcode::s_cbranch_scc0:
    case aco_opcode::s_cbranch_scc1:
    case aco_opcode::s_cbranch_vccz:
    case aco_opcode::s_cbranch_vccnz:
    case aco_opcode::s_cbranch_execz:
    case aco_opcode::s_cbranch_execnz:
       if (instr->salu().imm > block->index)
          return 0;
       FALLTHROUGH;
    case aco_opcode::s_swappc_b64:
    case aco_opcode::s_setpc_b64:
    case aco_opcode::s_call_b64:
       /* Restore defaults on loop back edges and calls. */
       return BITFIELD_MASK(mode_field_count);
    case aco_opcode::ds_cmpst_f32:
    case aco_opcode::ds_min_f32:
    case aco_opcode::ds_max_f32:
    case aco_opcode::ds_add_f32:
    case aco_opcode::ds_min_src2_f32:
    case aco_opcode::ds_max_src2_f32:
    case aco_opcode::ds_add_src2_f32:
    case aco_opcode::ds_cmpst_rtn_f32:
    case aco_opcode::ds_min_rtn_f32:
    case aco_opcode::ds_max_rtn_f32:
    case aco_opcode::ds_add_rtn_f32: return BITFIELD_BIT(mode_denorm32);
    case aco_opcode::ds_cmpst_f64:
    case aco_opcode::ds_min_f64:
    case aco_opcode::ds_max_f64:
    case aco_opcode::ds_min_src2_f64:
    case aco_opcode::ds_max_src2_f64:
    case aco_opcode::ds_cmpst_rtn_f64:
    case aco_opcode::ds_min_rtn_f64:
    case aco_opcode::ds_max_rtn_f64:
    case aco_opcode::ds_pk_add_f16:
    case aco_opcode::ds_pk_add_rtn_f16:
    case aco_opcode::ds_pk_add_bf16:
    case aco_opcode::ds_pk_add_rtn_bf16: return BITFIELD_BIT(mode_denorm16_64);
    case aco_opcode::v_cvt_pk_u8_f32: return BITFIELD_BIT(mode_round32);
    default: break;
    }

    if (!instr->isVALU() && !instr->isSALU() && !instr->isVINTRP())
       return 0;
    if (instr->definitions.empty())
       return 0;

    const aco_alu_opcode_info& info = instr_info.alu_opcode_infos[(int)instr->opcode];

    mode_mask res = 0;

    for (unsigned i = 0; i < info.num_operands; i++) {
       aco_type type = info.op_types[i];
       if (type.base_type != aco_base_type_float && type.base_type != aco_base_type_bfloat)
          continue;

       if (type.bit_size == 32)
          res |= BITFIELD_BIT(mode_denorm32);
       else if (type.bit_size >= 16)
          res |= BITFIELD_BIT(mode_denorm16_64);
    }

    aco_type type = info.def_types[0];
    if (type.base_type == aco_base_type_float || type.base_type == aco_base_type_bfloat) {
       if (type.bit_size == 32)
          res |= BITFIELD_BIT(mode_denorm32) | BITFIELD_BIT(mode_round32);
       else if (type.bit_size >= 16)
          res |= BITFIELD_BIT(mode_denorm16_64) | BITFIELD_BIT(mode_round16_64);

       if (type.bit_size <= 16)
          res |= BITFIELD_BIT(mode_fp16_ovfl);
    }

    if (instr->opcode == aco_opcode::v_fma_mixlo_f16 || instr->opcode == aco_opcode::v_fma_mixlo_f16)
       res |= BITFIELD_BIT(mode_round32);
    else if (instr->opcode == aco_opcode::v_fma_mix_f32 && instr->valu().opsel_hi)
       res |= BITFIELD_BIT(mode_denorm16_64);

    return res;
 }

 void
 emit_set_mode_block(fp_mode_ctx* ctx, Block* block)
 {
    Builder bld(ctx->program, block);
    fp_mode_state fp_state;
    const fp_mode_state default_state(block->fp_mode);

    if (block->index == 0) {
       bool inital_unknown = (ctx->program->info.merged_shader_compiled_separately &&
                              ctx->program->stage.sw == SWStage::GS) ||
                             (ctx->program->info.merged_shader_compiled_separately &&
                              ctx->program->stage.sw == SWStage::TCS);

       if (inital_unknown) {
          fp_state.dirty = BITFIELD_MASK(mode_field_count) & ~BITFIELD_BIT(mode_fp16_ovfl);
       } else {
          float_mode program_mode;
          program_mode.val = ctx->program->config->float_mode;
          fp_state = fp_mode_state(program_mode);
       }
    } else if (block->linear_preds.empty()) {
       fp_state = default_state;
    } else {
       assert(block->linear_preds[0] < block->index);
       fp_state = ctx->block_states[block->linear_preds[0]];
       for (unsigned i = 1; i < block->linear_preds.size(); i++) {
          unsigned pred = block->linear_preds[i];
          fp_mode_state other = pred < block->index
                                   ? ctx->block_states[pred]
                                   : fp_mode_state(ctx->program->blocks[pred].fp_mode);
          fp_state.join(other);
       }
    }

    /* If we don't know the value, set it to the default one next time. */
    u_foreach_bit (field, fp_state.dirty)
       fp_state.fields[field] = default_state.fields[field];

    for (std::vector<aco_ptr<Instruction>>::iterator it = block->instructions.begin();
         it < block->instructions.end(); ++it) {
       bool set_mode = false;

       Instruction* instr = it->get();

       if (instr->opcode == aco_opcode::p_v_cvt_f16_f32_rtne ||
           instr->opcode == aco_opcode::p_s_cvt_f16_f32_rtne) {
          set_mode |= fp_state.require(mode_round16_64, fp_round_ne);
          set_mode |= fp_state.require(mode_fp16_ovfl, default_state.fields[mode_fp16_ovfl]);
          set_mode |= fp_state.require(mode_denorm16_64, default_state.fields[mode_denorm16_64]);
          if (instr->opcode == aco_opcode::p_v_cvt_f16_f32_rtne)
             instr->opcode = aco_opcode::v_cvt_f16_f32;
          else
             instr->opcode = aco_opcode::s_cvt_f16_f32;
       } else if (instr->opcode == aco_opcode::p_v_cvt_pk_fp8_f32_ovfl) {
          set_mode |= fp_state.require(mode_fp16_ovfl, 1);
          instr->opcode = aco_opcode::v_cvt_pk_fp8_f32;
       } else {
          mode_mask default_needs = instr_default_needs(ctx, block, instr);
          u_foreach_bit (i, default_needs)
             set_mode |= fp_state.require((mode_field)i, default_state.fields[i]);
       }

       if (set_mode) {
          bld.reset(&block->instructions, it);
          emit_set_mode(bld, fp_state);
          fp_state.dirty = 0;
          /* Update the iterator if it was invalidated */
          it = bld.it;
       }
    }

    if (block->kind & block_kind_end_with_regs) {
       /* Restore default. */
       for (unsigned i = 0; i < mode_field_count; i++)
          fp_state.require((mode_field)i, default_state.fields[i]);
       if (fp_state.dirty) {
          bld.reset(block);
          emit_set_mode(bld, fp_state);
          fp_state.dirty = 0;
       }
    }

    ctx->block_states[block->index] = fp_state;
 }

 } // namespace

 bool
 instr_is_vmem_fp_atomic(Instruction* instr)
 {
    return vmem_default_needs(instr) != 0;
 }

 void
 insert_fp_mode(Program* program)
 {
    fp_mode_ctx ctx;
    ctx.program = program;
    ctx.block_states.resize(program->blocks.size());

    for (Block& block : program->blocks)
       emit_set_mode_block(&ctx, &block);
 }

 } // namespace aco
	/*
	* Copyright © 2025 Valve Corporation
	*
	* SPDX-License-Identifier: MIT
	*/

	#include "aco_builder.h"
	#include "aco_ir.h"

	#include <vector>

	namespace aco {

	namespace {

	enum mode_field : uint8_t {
	mode_round32 = 0,
	mode_round16_64,
	mode_denorm32,
	mode_denorm16_64,
	mode_fp16_ovfl,

	mode_field_count,
	};

	using mode_mask = uint8_t;
	static_assert(mode_field_count <= sizeof(mode_mask) * 8, "larger mode_mask needed");

	struct fp_mode_state {
	uint8_t fields[mode_field_count] = {};
	mode_mask dirty = 0; /* BITFIELD_BIT(enum mode_field) */

	fp_mode_state() = default;

	fp_mode_state(float_mode mode)
	{
	fields[mode_round32] = mode.round32;
	fields[mode_round16_64] = mode.round16_64;
	fields[mode_denorm32] = mode.denorm32;
	fields[mode_denorm16_64] = mode.denorm16_64;
	fields[mode_fp16_ovfl] = 0;
	}

	void join(const fp_mode_state& other)
	{
	dirty \|= other.dirty;
	for (unsigned i = 0; i < mode_field_count; i++) {
	if (fields[i] != other.fields[i])
	dirty \|= BITFIELD_BIT(i);
	}
	}

	bool require(mode_field field, uint8_t val)
	{
	if (fields[field] == val && !(dirty & BITFIELD_BIT(field)))
	return false;

	fields[field] = val;
	dirty \|= BITFIELD_BIT(field);
	return true;
	}

	uint8_t round() const { return fields[mode_round32] \| (fields[mode_round16_64] << 2); }

	uint8_t denorm() const { return fields[mode_denorm32] \| (fields[mode_denorm16_64] << 2); }
	};

	struct fp_mode_ctx {
	std::vector<fp_mode_state> block_states;
	Program* program;
	};

	void
	emit_set_mode(Builder& bld, const fp_mode_state& state)
	{
	bool set_round = state.dirty & (BITFIELD_BIT(mode_round32) \| BITFIELD_BIT(mode_round16_64));
	bool set_denorm = state.dirty & (BITFIELD_BIT(mode_denorm32) \| BITFIELD_BIT(mode_denorm16_64));
	bool set_fp16_ovfl = state.dirty & BITFIELD_BIT(mode_fp16_ovfl);

	if (bld.program->gfx_level >= GFX10) {
	if (set_round)
	bld.sopp(aco_opcode::s_round_mode, state.round());
	if (set_denorm)
	bld.sopp(aco_opcode::s_denorm_mode, state.denorm());
	} else if (set_round \|\| set_denorm) {
	/* "((size - 1) << 11) \| register" (MODE is encoded as register 1) */
	uint8_t val = state.round() \| (state.denorm() << 4);
	bld.sopk(aco_opcode::s_setreg_imm32_b32, Operand::literal32(val), (7 << 11) \| 1);
	}

	if (set_fp16_ovfl) {
	/* "((size - 1) << 11 \| (offset << 6) \| register" (MODE is encoded as register 1, we
	* want to set a single bit at offset 23)
	*/
	bld.sopk(aco_opcode::s_setreg_imm32_b32, Operand::literal32(state.fields[mode_fp16_ovfl]),
	(0 << 11) \| (23 << 6) \| 1);
	}
	}

	mode_mask
	vmem_default_needs(Instruction* instr)
	{
	switch (instr->opcode) {
	case aco_opcode::buffer_atomic_fcmpswap:
	case aco_opcode::buffer_atomic_fmin:
	case aco_opcode::buffer_atomic_fmax:
	case aco_opcode::buffer_atomic_add_f32:
	case aco_opcode::flat_atomic_fcmpswap:
	case aco_opcode::flat_atomic_fmin:
	case aco_opcode::flat_atomic_fmax:
	case aco_opcode::flat_atomic_add_f32:
	case aco_opcode::global_atomic_fcmpswap:
	case aco_opcode::global_atomic_fmin:
	case aco_opcode::global_atomic_fmax:
	case aco_opcode::global_atomic_add_f32:
	case aco_opcode::image_atomic_fcmpswap:
	case aco_opcode::image_atomic_fmin:
	case aco_opcode::image_atomic_fmax:
	case aco_opcode::image_atomic_add_flt: return BITFIELD_BIT(mode_denorm32);
	case aco_opcode::buffer_atomic_fcmpswap_x2:
	case aco_opcode::buffer_atomic_fmin_x2:
	case aco_opcode::buffer_atomic_fmax_x2:
	case aco_opcode::buffer_atomic_pk_add_f16:
	case aco_opcode::buffer_atomic_pk_add_bf16:
	case aco_opcode::flat_atomic_fcmpswap_x2:
	case aco_opcode::flat_atomic_fmin_x2:
	case aco_opcode::flat_atomic_fmax_x2:
	case aco_opcode::flat_atomic_pk_add_f16:
	case aco_opcode::flat_atomic_pk_add_bf16:
	case aco_opcode::global_atomic_fcmpswap_x2:
	case aco_opcode::global_atomic_fmin_x2:
	case aco_opcode::global_atomic_fmax_x2:
	case aco_opcode::global_atomic_pk_add_f16:
	case aco_opcode::global_atomic_pk_add_bf16:
	case aco_opcode::image_atomic_pk_add_f16:
	case aco_opcode::image_atomic_pk_add_bf16: return BITFIELD_BIT(mode_denorm16_64);
	default: return 0;
	}
	}

	mode_mask
	instr_default_needs(fp_mode_ctx* ctx, Block* block, Instruction* instr)
	{
	if ((instr->isVMEM() \|\| instr->isFlatLike()) && ctx->program->gfx_level < GFX12)
	return vmem_default_needs(instr);

	switch (instr->opcode) {
	case aco_opcode::s_branch:
	case aco_opcode::s_cbranch_scc0:
	case aco_opcode::s_cbranch_scc1:
	case aco_opcode::s_cbranch_vccz:
	case aco_opcode::s_cbranch_vccnz:
	case aco_opcode::s_cbranch_execz:
	case aco_opcode::s_cbranch_execnz:
	if (instr->salu().imm > block->index)
	return 0;
	FALLTHROUGH;
	case aco_opcode::s_swappc_b64:
	case aco_opcode::s_setpc_b64:
	case aco_opcode::s_call_b64:
	/* Restore defaults on loop back edges and calls. */
	return BITFIELD_MASK(mode_field_count);
	case aco_opcode::ds_cmpst_f32:
	case aco_opcode::ds_min_f32:
	case aco_opcode::ds_max_f32:
	case aco_opcode::ds_add_f32:
	case aco_opcode::ds_min_src2_f32:
	case aco_opcode::ds_max_src2_f32:
	case aco_opcode::ds_add_src2_f32:
	case aco_opcode::ds_cmpst_rtn_f32:
	case aco_opcode::ds_min_rtn_f32:
	case aco_opcode::ds_max_rtn_f32:
	case aco_opcode::ds_add_rtn_f32: return BITFIELD_BIT(mode_denorm32);
	case aco_opcode::ds_cmpst_f64:
	case aco_opcode::ds_min_f64:
	case aco_opcode::ds_max_f64:
	case aco_opcode::ds_min_src2_f64:
	case aco_opcode::ds_max_src2_f64:
	case aco_opcode::ds_cmpst_rtn_f64:
	case aco_opcode::ds_min_rtn_f64:
	case aco_opcode::ds_max_rtn_f64:
	case aco_opcode::ds_pk_add_f16:
	case aco_opcode::ds_pk_add_rtn_f16:
	case aco_opcode::ds_pk_add_bf16:
	case aco_opcode::ds_pk_add_rtn_bf16: return BITFIELD_BIT(mode_denorm16_64);
	case aco_opcode::v_cvt_pk_u8_f32: return BITFIELD_BIT(mode_round32);
	default: break;
	}

	if (!instr->isVALU() && !instr->isSALU() && !instr->isVINTRP())
	return 0;
	if (instr->definitions.empty())
	return 0;

	const aco_alu_opcode_info& info = instr_info.alu_opcode_infos[(int)instr->opcode];

	mode_mask res = 0;

	for (unsigned i = 0; i < info.num_operands; i++) {
	aco_type type = info.op_types[i];
	if (type.base_type != aco_base_type_float && type.base_type != aco_base_type_bfloat)
	continue;

	if (type.bit_size == 32)
	res \|= BITFIELD_BIT(mode_denorm32);
	else if (type.bit_size >= 16)
	res \|= BITFIELD_BIT(mode_denorm16_64);
	}

	aco_type type = info.def_types[0];
	if (type.base_type == aco_base_type_float \|\| type.base_type == aco_base_type_bfloat) {
	if (type.bit_size == 32)
	res \|= BITFIELD_BIT(mode_denorm32) \| BITFIELD_BIT(mode_round32);
	else if (type.bit_size >= 16)
	res \|= BITFIELD_BIT(mode_denorm16_64) \| BITFIELD_BIT(mode_round16_64);

	if (type.bit_size <= 16)
	res \|= BITFIELD_BIT(mode_fp16_ovfl);
	}

	if (instr->opcode == aco_opcode::v_fma_mixlo_f16 \|\| instr->opcode == aco_opcode::v_fma_mixlo_f16)
	res \|= BITFIELD_BIT(mode_round32);
	else if (instr->opcode == aco_opcode::v_fma_mix_f32 && instr->valu().opsel_hi)
	res \|= BITFIELD_BIT(mode_denorm16_64);

	return res;
	}

	void
	emit_set_mode_block(fp_mode_ctx* ctx, Block* block)
	{
	Builder bld(ctx->program, block);
	fp_mode_state fp_state;
	const fp_mode_state default_state(block->fp_mode);

	if (block->index == 0) {
	bool inital_unknown = (ctx->program->info.merged_shader_compiled_separately &&
	ctx->program->stage.sw == SWStage::GS) \|\|
	(ctx->program->info.merged_shader_compiled_separately &&
	ctx->program->stage.sw == SWStage::TCS);

	if (inital_unknown) {
	fp_state.dirty = BITFIELD_MASK(mode_field_count) & ~BITFIELD_BIT(mode_fp16_ovfl);
	} else {
	float_mode program_mode;
	program_mode.val = ctx->program->config->float_mode;
	fp_state = fp_mode_state(program_mode);
	}
	} else if (block->linear_preds.empty()) {
	fp_state = default_state;
	} else {
	assert(block->linear_preds[0] < block->index);
	fp_state = ctx->block_states[block->linear_preds[0]];
	for (unsigned i = 1; i < block->linear_preds.size(); i++) {
	unsigned pred = block->linear_preds[i];
	fp_mode_state other = pred < block->index
	? ctx->block_states[pred]
	: fp_mode_state(ctx->program->blocks[pred].fp_mode);
	fp_state.join(other);
	}
	}

	/* If we don't know the value, set it to the default one next time. */
	u_foreach_bit (field, fp_state.dirty)
	fp_state.fields[field] = default_state.fields[field];

	for (std::vector<aco_ptr<Instruction>>::iterator it = block->instructions.begin();
	it < block->instructions.end(); ++it) {
	bool set_mode = false;

	Instruction* instr = it->get();

	if (instr->opcode == aco_opcode::p_v_cvt_f16_f32_rtne \|\|
	instr->opcode == aco_opcode::p_s_cvt_f16_f32_rtne) {
	set_mode \|= fp_state.require(mode_round16_64, fp_round_ne);
	set_mode \|= fp_state.require(mode_fp16_ovfl, default_state.fields[mode_fp16_ovfl]);
	set_mode \|= fp_state.require(mode_denorm16_64, default_state.fields[mode_denorm16_64]);
	if (instr->opcode == aco_opcode::p_v_cvt_f16_f32_rtne)
	instr->opcode = aco_opcode::v_cvt_f16_f32;
	else
	instr->opcode = aco_opcode::s_cvt_f16_f32;
	} else if (instr->opcode == aco_opcode::p_v_cvt_pk_fp8_f32_ovfl) {
	set_mode \|= fp_state.require(mode_fp16_ovfl, 1);
	instr->opcode = aco_opcode::v_cvt_pk_fp8_f32;
	} else {
	mode_mask default_needs = instr_default_needs(ctx, block, instr);
	u_foreach_bit (i, default_needs)
	set_mode \|= fp_state.require((mode_field)i, default_state.fields[i]);
	}

	if (set_mode) {
	bld.reset(&block->instructions, it);
	emit_set_mode(bld, fp_state);
	fp_state.dirty = 0;
	/* Update the iterator if it was invalidated */
	it = bld.it;
	}
	}

	if (block->kind & block_kind_end_with_regs) {
	/* Restore default. */
	for (unsigned i = 0; i < mode_field_count; i++)
	fp_state.require((mode_field)i, default_state.fields[i]);
	if (fp_state.dirty) {
	bld.reset(block);
	emit_set_mode(bld, fp_state);
	fp_state.dirty = 0;
	}
	}

	ctx->block_states[block->index] = fp_state;
	}

	} // namespace

	bool
	instr_is_vmem_fp_atomic(Instruction* instr)
	{
	return vmem_default_needs(instr) != 0;
	}

	void
	insert_fp_mode(Program* program)
	{
	fp_mode_ctx ctx;
	ctx.program = program;
	ctx.block_states.resize(program->blocks.size());

	for (Block& block : program->blocks)
	emit_set_mode_block(&ctx, &block);
	}

	} // namespace aco