src/amd/common/nir/ac_nir_helpers.h - third_party/mesa - Git at Google

 /*
  * Copyright © 2024 Valve Corporation
  *
  * SPDX-License-Identifier: MIT
  */


 #ifndef AC_NIR_HELPERS_H
 #define AC_NIR_HELPERS_H

 #include "ac_hw_stage.h"
 #include "ac_shader_args.h"
 #include "ac_shader_util.h"
 #include "ac_nir.h"
 #include "nir_defines.h"

 #ifdef __cplusplus
 extern "C" {
 #endif

 #define AC_NIR_STORE_IO(b, store_val, const_offset, write_mask, hi_16bit, func, ...) \
    do { \
       if ((store_val)->bit_size >= 32) { \
          const unsigned store_write_mask = (write_mask); \
          const unsigned store_const_offset = (const_offset); \
          func((b), (store_val), __VA_ARGS__); \
       } else { \
          u_foreach_bit(c, (write_mask)) { \
             const unsigned store_write_mask = 1; \
             const unsigned store_const_offset = (const_offset) + c * 4 + ((hi_16bit) ? 2 : 0); \
             nir_def *store_component = nir_channel(b, (store_val), c); \
             func((b), store_component, __VA_ARGS__); \
          } \
       } \
    } while (0)

 #define AC_NIR_LOAD_IO(load, b, num_components, bit_size, hi_16bit, func, ...) \
    do { \
       const unsigned load_bit_size = MAX2(32, (bit_size)); \
       (load) = func((b), (num_components), load_bit_size, __VA_ARGS__); \
       if ((bit_size) < load_bit_size) { \
          if ((hi_16bit)) { \
             (load) = nir_unpack_32_2x16_split_y(b, load); \
          } else { \
             (load) = nir_unpack_32_2x16_split_x(b, load); \
          } \
       } \
    } while (0)

 typedef struct
 {
    /* GS output stream index, 2 bit per component */
    uint8_t stream;
    /* Bitmask of components used: 4 bits per slot, 1 bit per component. */
    uint8_t components_mask : 4;
    /* Bitmask of components that are used as varying, 1 bit per component. */
    uint8_t as_varying_mask : 4;
    /* Bitmask of components that are used as sysval, 1 bit per component. */
    uint8_t as_sysval_mask : 4;
    /* Prefix sum over all component masks. Used by the GS outputs in LDS for NGG GS.
     * This is set even if components_mask is 0, in which case it's the offset after the last output.
     */
    uint16_t packed_slot_gs_out_offset : 12;
    /* Prefix sum over all component masks. Used by XFB outputs in LDS for NGG VS and TES.
     * This is set even if xfb_components_mask is 0, in which case it's the offset after the last output.
     * For NGG GS, it's equal to packed_slot_gs_out_offset because NGG GS has all outputs in LDS.
     */
    uint16_t packed_slot_xfb_lds_offset : 12;
    /* Bitmask of components written by XFB: 4 bits per slot, 1 bit per component.
     * For NGG GS, it's equal to components_mask because NGG GS has all outputs in LDS.
     */
    uint8_t xfb_lds_components_mask : 4;
    /* Bitmask of components that store a constant value. */
    uint8_t const_mask : 4;
    /* Bitmask of components that store a non-constant value at least once. Used only during gathering. */
    uint8_t nonconst_mask : 4;
 } ac_nir_prerast_per_output_info;

 typedef struct
 {
    /* Low and high 16 bits are packed into 32 bits. */
    nir_def *outputs[NUM_TOTAL_VARYING_SLOTS][4];
    uint32_t const_values[NUM_TOTAL_VARYING_SLOTS][4];

    nir_alu_type types[NUM_TOTAL_VARYING_SLOTS][4];
    ac_nir_prerast_per_output_info infos[NUM_TOTAL_VARYING_SLOTS];

    /* The size of all components, packed. */
    uint16_t total_packed_gs_out_size;
    uint16_t total_packed_xfb_lds_size;
 } ac_nir_prerast_out;

 typedef struct {
    nir_def *num_repacked_invocations;
    nir_def *repacked_invocation_index;
 } ac_nir_wg_repack_result;

 /* Maps I/O semantics to the actual location used by the lowering pass. */
 typedef unsigned (*ac_nir_map_io_driver_location)(unsigned semantic);

 /* Forward declaration of nir_builder so we don't have to include nir_builder.h here */
 struct nir_builder;
 typedef struct nir_builder nir_builder;

 struct nir_xfb_info;
 typedef struct nir_xfb_info nir_xfb_info;

 /* Executed by ac_nir_cull when the current primitive is accepted. */
 typedef void (*ac_nir_cull_accepted)(nir_builder *b, void *state);

 nir_def *
 ac_nir_unpack_value(nir_builder *b, nir_def *value, unsigned rshift, unsigned bitwidth);

 void
 ac_nir_store_var_components(nir_builder *b, nir_variable *var, nir_def *value,
                             unsigned component, unsigned writemask);

 void
 ac_nir_gather_prerast_store_output_info(nir_builder *b, nir_intrinsic_instr *intrin,
                                         ac_nir_prerast_out *out, bool gather_values);

 void
 ac_nir_export_primitive(nir_builder *b, nir_def *prim, nir_def *row);

 void
 ac_nir_export_position(nir_builder *b,
                        enum amd_gfx_level gfx_level,
                        uint32_t export_clipdist_mask,
                        bool dont_export_cull_distances,
                        bool write_pos_to_clipvertex,
                        bool no_param_export,
                        bool force_vrs,
                        uint64_t outputs_written,
                        ac_nir_prerast_out *out,
                        nir_def *row);

 void
 ac_nir_export_parameters(nir_builder *b,
                          const uint8_t *param_offsets,
                          uint64_t outputs_written,
                          uint16_t outputs_written_16bit,
                          ac_nir_prerast_out *out);

 void
 ac_nir_store_parameters_to_attr_ring(nir_builder *b,
                                      const uint8_t *param_offsets,
                                      const uint64_t outputs_written,
                                      const uint16_t outputs_written_16bit,
                                      ac_nir_prerast_out *out,
                                      nir_def *num_export_threads_in_wave);

 nir_def *
 ac_nir_calc_io_off(nir_builder *b, unsigned component, nir_def *io_offset, nir_def *base_stride,
                    unsigned component_stride, unsigned mapped_driver_location);

 unsigned
 ac_nir_map_io_location(unsigned location,
                        uint64_t mask,
                        ac_nir_map_io_driver_location map_io);

 nir_def *
 ac_nir_cull_primitive(nir_builder *b,
                       bool skip_viewport_state_culling,
                       bool use_point_tri_intersection,
                       nir_def *initially_accepted,
                       nir_def *pos[3][4],
                       unsigned num_vertices,
                       ac_nir_cull_accepted accept_func,
                       void *state);

 void
 ac_nir_sleep(nir_builder *b, unsigned num_cycles);

 nir_def *
 ac_average_samples(nir_builder *b, nir_def **samples, unsigned num_samples);

 void
 ac_optimization_barrier_vgpr_array(const struct radeon_info *info, nir_builder *b,
                                    nir_def **array, unsigned num_elements,
                                    unsigned num_components);

 nir_def *
 ac_get_global_ids(nir_builder *b, unsigned num_components, unsigned bit_size);

 void
 ac_nir_emit_legacy_streamout(nir_builder *b, unsigned stream, nir_xfb_info *info, ac_nir_prerast_out *out);

 bool
 ac_nir_gs_shader_query(nir_builder *b,
                        bool has_gen_prim_query,
                        bool has_gs_invocations_query,
                        bool has_gs_primitives_query,
                        unsigned num_vertices_per_primitive,
                        unsigned wave_size,
                        nir_def *vertex_count[4],
                        nir_def *primitive_count[4]);

 nir_def *
 ac_nir_pack_ngg_prim_exp_arg(nir_builder *b, unsigned num_vertices_per_primitives,
                              nir_def *vertex_indices[3], nir_def *is_null_prim,
                              enum amd_gfx_level gfx_level);

 void
 ac_nir_clamp_vertex_color_outputs(nir_builder *b, ac_nir_prerast_out *out);

 void
 ac_nir_ngg_alloc_vertices_and_primitives(nir_builder *b,
                                          nir_def *num_vtx,
                                          nir_def *num_prim,
                                          bool fully_culled_workaround);

 void
 ac_nir_create_output_phis(nir_builder *b,
                           const uint64_t outputs_written,
                           const uint64_t outputs_written_16bit,
                           ac_nir_prerast_out *out);

 void
 ac_nir_ngg_build_streamout_buffer_info(nir_builder *b,
                                        nir_xfb_info *info,
                                        enum amd_gfx_level gfx_level,
                                        bool has_xfb_prim_query,
                                        bool use_gfx12_xfb_intrinsic,
                                        nir_def *scratch_base,
                                        nir_def *tid_in_tg,
                                        nir_def *gen_prim[4],
                                        nir_def *so_buffer_ret[4],
                                        nir_def *buffer_offsets_ret[4],
                                        nir_def *emit_prim_ret[4]);

 bool
 ac_nir_is_const_output(ac_nir_prerast_out *pr_out, gl_varying_slot slot, unsigned component);

 nir_def *
 ac_nir_get_const_output(nir_builder *b, ac_nir_prerast_out *pr_out, gl_varying_slot slot,
                         unsigned component);

 void
 ac_nir_store_shared_xfb(nir_builder *b, nir_def *value, nir_def *vtxptr, ac_nir_prerast_out *pr_out,
                         gl_varying_slot slot, unsigned component);

 nir_def *
 ac_nir_load_shared_xfb(nir_builder *b, nir_def *vtxptr, ac_nir_prerast_out *pr_out,
                        gl_varying_slot slot, unsigned component);

 void
 ac_nir_store_shared_gs_out(nir_builder *b, nir_def *value, nir_def *vtxptr, ac_nir_prerast_out *pr_out,
                            gl_varying_slot slot, unsigned component);

 nir_def *
 ac_nir_load_shared_gs_out(nir_builder *b, nir_def *vtxptr, ac_nir_prerast_out *pr_out,
                           gl_varying_slot slot, unsigned component);

 void
 ac_nir_ngg_build_streamout_vertex(nir_builder *b, nir_xfb_info *info,
                                   unsigned stream, nir_def *so_buffer[4],
                                   nir_def *buffer_offsets[4],
                                   unsigned vertex_index, nir_def *vtx_lds_addr,
                                   ac_nir_prerast_out *pr_out);

 void
 ac_nir_repack_invocations_in_workgroup(nir_builder *b, nir_def **input_bool,
                                        ac_nir_wg_repack_result *results, const unsigned num_repacks,
                                        nir_def *lds_addr_base, unsigned max_num_waves,
                                        unsigned wave_size);

 void
 ac_nir_compute_prerast_packed_output_info(ac_nir_prerast_out *pr_out);

 unsigned
 ac_nir_gs_output_component_mask_with_stream(ac_nir_prerast_per_output_info *info, unsigned stream);

 nir_shader *
 ac_nir_create_gs_copy_shader(const nir_shader *gs_nir, ac_nir_lower_legacy_gs_options *options,
                              ac_nir_prerast_out *out);

 #ifdef __cplusplus
 }
 #endif

 #endif /* AC_NIR_HELPERS_H */
	/*
	* Copyright © 2024 Valve Corporation
	*
	* SPDX-License-Identifier: MIT
	*/


	#ifndef AC_NIR_HELPERS_H
	#define AC_NIR_HELPERS_H

	#include "ac_hw_stage.h"
	#include "ac_shader_args.h"
	#include "ac_shader_util.h"
	#include "ac_nir.h"
	#include "nir_defines.h"

	#ifdef __cplusplus
	extern "C" {
	#endif

	#define AC_NIR_STORE_IO(b, store_val, const_offset, write_mask, hi_16bit, func, ...) \
	do { \
	if ((store_val)->bit_size >= 32) { \
	const unsigned store_write_mask = (write_mask); \
	const unsigned store_const_offset = (const_offset); \
	func((b), (store_val), __VA_ARGS__); \
	} else { \
	u_foreach_bit(c, (write_mask)) { \
	const unsigned store_write_mask = 1; \
	const unsigned store_const_offset = (const_offset) + c * 4 + ((hi_16bit) ? 2 : 0); \
	nir_def *store_component = nir_channel(b, (store_val), c); \
	func((b), store_component, __VA_ARGS__); \
	} \
	} \
	} while (0)

	#define AC_NIR_LOAD_IO(load, b, num_components, bit_size, hi_16bit, func, ...) \
	do { \
	const unsigned load_bit_size = MAX2(32, (bit_size)); \
	(load) = func((b), (num_components), load_bit_size, __VA_ARGS__); \
	if ((bit_size) < load_bit_size) { \
	if ((hi_16bit)) { \
	(load) = nir_unpack_32_2x16_split_y(b, load); \
	} else { \
	(load) = nir_unpack_32_2x16_split_x(b, load); \
	} \
	} \
	} while (0)

	typedef struct
	{
	/* GS output stream index, 2 bit per component */
	uint8_t stream;
	/* Bitmask of components used: 4 bits per slot, 1 bit per component. */
	uint8_t components_mask : 4;
	/* Bitmask of components that are used as varying, 1 bit per component. */
	uint8_t as_varying_mask : 4;
	/* Bitmask of components that are used as sysval, 1 bit per component. */
	uint8_t as_sysval_mask : 4;
	/* Prefix sum over all component masks. Used by the GS outputs in LDS for NGG GS.
	* This is set even if components_mask is 0, in which case it's the offset after the last output.
	*/
	uint16_t packed_slot_gs_out_offset : 12;
	/* Prefix sum over all component masks. Used by XFB outputs in LDS for NGG VS and TES.
	* This is set even if xfb_components_mask is 0, in which case it's the offset after the last output.
	* For NGG GS, it's equal to packed_slot_gs_out_offset because NGG GS has all outputs in LDS.
	*/
	uint16_t packed_slot_xfb_lds_offset : 12;
	/* Bitmask of components written by XFB: 4 bits per slot, 1 bit per component.
	* For NGG GS, it's equal to components_mask because NGG GS has all outputs in LDS.
	*/
	uint8_t xfb_lds_components_mask : 4;
	/* Bitmask of components that store a constant value. */
	uint8_t const_mask : 4;
	/* Bitmask of components that store a non-constant value at least once. Used only during gathering. */
	uint8_t nonconst_mask : 4;
	} ac_nir_prerast_per_output_info;

	typedef struct
	{
	/* Low and high 16 bits are packed into 32 bits. */
	nir_def *outputs[NUM_TOTAL_VARYING_SLOTS][4];
	uint32_t const_values[NUM_TOTAL_VARYING_SLOTS][4];

	nir_alu_type types[NUM_TOTAL_VARYING_SLOTS][4];
	ac_nir_prerast_per_output_info infos[NUM_TOTAL_VARYING_SLOTS];

	/* The size of all components, packed. */
	uint16_t total_packed_gs_out_size;
	uint16_t total_packed_xfb_lds_size;
	} ac_nir_prerast_out;

	typedef struct {
	nir_def *num_repacked_invocations;
	nir_def *repacked_invocation_index;
	} ac_nir_wg_repack_result;

	/* Maps I/O semantics to the actual location used by the lowering pass. */
	typedef unsigned (*ac_nir_map_io_driver_location)(unsigned semantic);

	/* Forward declaration of nir_builder so we don't have to include nir_builder.h here */
	struct nir_builder;
	typedef struct nir_builder nir_builder;

	struct nir_xfb_info;
	typedef struct nir_xfb_info nir_xfb_info;

	/* Executed by ac_nir_cull when the current primitive is accepted. */
	typedef void (ac_nir_cull_accepted)(nir_builder b, void *state);

	nir_def *
	ac_nir_unpack_value(nir_builder b, nir_def value, unsigned rshift, unsigned bitwidth);

	void
	ac_nir_store_var_components(nir_builder b, nir_variable var, nir_def *value,
	unsigned component, unsigned writemask);

	void
	ac_nir_gather_prerast_store_output_info(nir_builder b, nir_intrinsic_instr intrin,
	ac_nir_prerast_out *out, bool gather_values);

	void
	ac_nir_export_primitive(nir_builder b, nir_def prim, nir_def *row);

	void
	ac_nir_export_position(nir_builder *b,
	enum amd_gfx_level gfx_level,
	uint32_t export_clipdist_mask,
	bool dont_export_cull_distances,
	bool write_pos_to_clipvertex,
	bool no_param_export,
	bool force_vrs,
	uint64_t outputs_written,
	ac_nir_prerast_out *out,
	nir_def *row);

	void
	ac_nir_export_parameters(nir_builder *b,
	const uint8_t *param_offsets,
	uint64_t outputs_written,
	uint16_t outputs_written_16bit,
	ac_nir_prerast_out *out);

	void
	ac_nir_store_parameters_to_attr_ring(nir_builder *b,
	const uint8_t *param_offsets,
	const uint64_t outputs_written,
	const uint16_t outputs_written_16bit,
	ac_nir_prerast_out *out,
	nir_def *num_export_threads_in_wave);

	nir_def *
	ac_nir_calc_io_off(nir_builder b, unsigned component, nir_def io_offset, nir_def *base_stride,
	unsigned component_stride, unsigned mapped_driver_location);

	unsigned
	ac_nir_map_io_location(unsigned location,
	uint64_t mask,
	ac_nir_map_io_driver_location map_io);

	nir_def *
	ac_nir_cull_primitive(nir_builder *b,
	bool skip_viewport_state_culling,
	bool use_point_tri_intersection,
	nir_def *initially_accepted,
	nir_def *pos[3][4],
	unsigned num_vertices,
	ac_nir_cull_accepted accept_func,
	void *state);

	void
	ac_nir_sleep(nir_builder *b, unsigned num_cycles);

	nir_def *
	ac_average_samples(nir_builder b, nir_def *samples, unsigned num_samples);

	void
	ac_optimization_barrier_vgpr_array(const struct radeon_info info, nir_builder b,
	nir_def **array, unsigned num_elements,
	unsigned num_components);

	nir_def *
	ac_get_global_ids(nir_builder *b, unsigned num_components, unsigned bit_size);

	void
	ac_nir_emit_legacy_streamout(nir_builder b, unsigned stream, nir_xfb_info info, ac_nir_prerast_out *out);

	bool
	ac_nir_gs_shader_query(nir_builder *b,
	bool has_gen_prim_query,
	bool has_gs_invocations_query,
	bool has_gs_primitives_query,
	unsigned num_vertices_per_primitive,
	unsigned wave_size,
	nir_def *vertex_count[4],
	nir_def *primitive_count[4]);

	nir_def *
	ac_nir_pack_ngg_prim_exp_arg(nir_builder *b, unsigned num_vertices_per_primitives,
	nir_def vertex_indices[3], nir_def is_null_prim,
	enum amd_gfx_level gfx_level);

	void
	ac_nir_clamp_vertex_color_outputs(nir_builder b, ac_nir_prerast_out out);

	void
	ac_nir_ngg_alloc_vertices_and_primitives(nir_builder *b,
	nir_def *num_vtx,
	nir_def *num_prim,
	bool fully_culled_workaround);

	void
	ac_nir_create_output_phis(nir_builder *b,
	const uint64_t outputs_written,
	const uint64_t outputs_written_16bit,
	ac_nir_prerast_out *out);

	void
	ac_nir_ngg_build_streamout_buffer_info(nir_builder *b,
	nir_xfb_info *info,
	enum amd_gfx_level gfx_level,
	bool has_xfb_prim_query,
	bool use_gfx12_xfb_intrinsic,
	nir_def *scratch_base,
	nir_def *tid_in_tg,
	nir_def *gen_prim[4],
	nir_def *so_buffer_ret[4],
	nir_def *buffer_offsets_ret[4],
	nir_def *emit_prim_ret[4]);

	bool
	ac_nir_is_const_output(ac_nir_prerast_out *pr_out, gl_varying_slot slot, unsigned component);

	nir_def *
	ac_nir_get_const_output(nir_builder b, ac_nir_prerast_out pr_out, gl_varying_slot slot,
	unsigned component);

	void
	ac_nir_store_shared_xfb(nir_builder b, nir_def value, nir_def vtxptr, ac_nir_prerast_out pr_out,
	gl_varying_slot slot, unsigned component);

	nir_def *
	ac_nir_load_shared_xfb(nir_builder b, nir_def vtxptr, ac_nir_prerast_out *pr_out,
	gl_varying_slot slot, unsigned component);

	void
	ac_nir_store_shared_gs_out(nir_builder b, nir_def value, nir_def vtxptr, ac_nir_prerast_out pr_out,
	gl_varying_slot slot, unsigned component);

	nir_def *
	ac_nir_load_shared_gs_out(nir_builder b, nir_def vtxptr, ac_nir_prerast_out *pr_out,
	gl_varying_slot slot, unsigned component);

	void
	ac_nir_ngg_build_streamout_vertex(nir_builder b, nir_xfb_info info,
	unsigned stream, nir_def *so_buffer[4],
	nir_def *buffer_offsets[4],
	unsigned vertex_index, nir_def *vtx_lds_addr,
	ac_nir_prerast_out *pr_out);

	void
	ac_nir_repack_invocations_in_workgroup(nir_builder b, nir_def *input_bool,
	ac_nir_wg_repack_result *results, const unsigned num_repacks,
	nir_def *lds_addr_base, unsigned max_num_waves,
	unsigned wave_size);

	void
	ac_nir_compute_prerast_packed_output_info(ac_nir_prerast_out *pr_out);

	unsigned
	ac_nir_gs_output_component_mask_with_stream(ac_nir_prerast_per_output_info *info, unsigned stream);

	nir_shader *
	ac_nir_create_gs_copy_shader(const nir_shader gs_nir, ac_nir_lower_legacy_gs_options options,
	ac_nir_prerast_out *out);

	#ifdef __cplusplus
	}
	#endif

	#endif /* AC_NIR_HELPERS_H */