src/microsoft/clc/clc_nir.c - third_party/mesa - Git at Google

 /*
  * Copyright © Microsoft Corporation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
  * to deal in the Software without restriction, including without limitation
  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  * and/or sell copies of the Software, and to permit persons to whom the
  * Software is furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice (including the next
  * paragraph) shall be included in all copies or substantial portions of the
  * Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  * IN THE SOFTWARE.
  */

 #include "util/u_math.h"
 #include "nir.h"
 #include "glsl_types.h"
 #include "nir_builder.h"
 #include "nir_deref.h"

 #include "clc_nir.h"
 #include "clc_compiler.h"
 #include "../compiler/dxil_nir.h"

 static nir_def *
 load_ubo(nir_builder *b, nir_intrinsic_instr *intr, nir_variable *var, unsigned offset)
 {
    return nir_load_ubo(b,
                        intr->def.num_components,
                        intr->def.bit_size,
                        nir_imm_int(b, var->data.binding),
                        nir_imm_int(b, offset),
                        .align_mul = 256,
                        .align_offset = offset,
                        .range_base = offset,
                        .range = intr->def.bit_size * intr->def.num_components / 8);
 }

 static bool
 is_clc_system_value(const nir_instr *instr, const void *state)
 {
    if (instr->type != nir_instr_type_intrinsic)
       return false;
    nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
    switch (intr->intrinsic) {
    case nir_intrinsic_load_base_global_invocation_id:
    case nir_intrinsic_load_work_dim:
    case nir_intrinsic_load_num_workgroups:
    case nir_intrinsic_load_base_workgroup_id:
       return true;
    default:
       return false;
    }
 }

 static nir_def *
 lower_clc_system_value(nir_builder *b, nir_instr *instr, void *state)
 {
    nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
    nir_variable *var = (nir_variable *)state;

    switch (intr->intrinsic) {
    case nir_intrinsic_load_base_global_invocation_id:
       return load_ubo(b, intr, var, offsetof(struct clc_work_properties_data, global_offset_x));
    case nir_intrinsic_load_work_dim:
       return load_ubo(b, intr, var, offsetof(struct clc_work_properties_data, work_dim));
    case nir_intrinsic_load_num_workgroups:
       return load_ubo(b, intr, var, offsetof(struct clc_work_properties_data, group_count_total_x));
    case nir_intrinsic_load_base_workgroup_id:
       return load_ubo(b, intr, var, offsetof(struct clc_work_properties_data, group_id_offset_x));
    default:
       return NULL;
    }
 }

 bool
 clc_nir_lower_system_values(nir_shader *nir, nir_variable *var)
 {
    return nir_shader_lower_instructions(nir, is_clc_system_value, lower_clc_system_value, var);
 }

 static bool
 is_load_kernel_input(const nir_instr *instr, const void *state)
 {
    return instr->type == nir_instr_type_intrinsic &&
       nir_instr_as_intrinsic(instr)->intrinsic == nir_intrinsic_load_kernel_input;
 }

 static nir_def *
 lower_load_kernel_input(nir_builder *b, nir_instr *instr, void *state)
 {
    nir_variable *var = (nir_variable *)state;
    nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);

    unsigned bit_size = intr->def.bit_size;
    enum glsl_base_type base_type;

    switch (bit_size) {
    case 64:
       base_type = GLSL_TYPE_UINT64;
       break;
    case 32:
       base_type = GLSL_TYPE_UINT;
       break;
    case 16:
       base_type = GLSL_TYPE_UINT16;
       break;
    case 8:
       base_type = GLSL_TYPE_UINT8;
       break;
    default:
       UNREACHABLE("invalid bit size");
    }

    const struct glsl_type *type =
       glsl_vector_type(base_type, intr->def.num_components);
    nir_def *ptr = nir_vec2(b, nir_imm_int(b, var->data.binding),
                                   nir_u2uN(b, intr->src[0].ssa, 32));
    nir_deref_instr *deref = nir_build_deref_cast(b, ptr, nir_var_mem_ubo, type,
                                                     bit_size / 8);
    deref->cast.align_mul = nir_intrinsic_align_mul(intr);
    deref->cast.align_offset = nir_intrinsic_align_offset(intr);

    return nir_load_deref(b, deref);
 }

 bool
 clc_nir_lower_kernel_input_loads(nir_shader *nir, nir_variable *var)
 {
    return nir_shader_lower_instructions(nir, is_load_kernel_input, lower_load_kernel_input, var);
 }


 static nir_variable *
 add_printf_var(struct nir_shader *nir, unsigned uav_id)
 {
    /* This size is arbitrary. Minimum required per spec is 1MB */
    const unsigned max_printf_size = 1 * 1024 * 1024;
    const unsigned printf_array_size = max_printf_size / sizeof(unsigned);
    nir_variable *var =
       nir_variable_create(nir, nir_var_mem_ssbo,
                           glsl_array_type(glsl_uint_type(), printf_array_size, sizeof(unsigned)),
                           "printf");
    var->data.binding = uav_id;
    return var;
 }

 bool
 clc_lower_printf_base(nir_shader *nir, unsigned uav_id)
 {
    nir_variable *printf_var = NULL;
    nir_def *printf_deref = NULL;
    nir_foreach_function_impl(impl, nir) {
       nir_builder b = nir_builder_at(nir_before_impl(impl));
       bool progress = false;

       nir_foreach_block(block, impl) {
          nir_foreach_instr_safe(instr, block) {
             if (instr->type != nir_instr_type_intrinsic)
                continue;
             nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
             if (intrin->intrinsic != nir_intrinsic_load_printf_buffer_address)
                continue;

             if (!printf_var) {
                printf_var = add_printf_var(nir, uav_id);
                nir_deref_instr *deref = nir_build_deref_var(&b, printf_var);
                printf_deref = &deref->def;
             }
             nir_def_rewrite_uses(&intrin->def, printf_deref);
             progress = true;
          }
       }

       nir_progress(progress, impl,
                    nir_metadata_loop_analysis | nir_metadata_block_index | nir_metadata_dominance);
    }

    return printf_var != NULL;
 }

 /* Find patterns of:
  * - deref_var for one of the kernel inputs
  * - load_deref to get a pointer to global/constant memory
  * - cast the pointer into a deref
  * - use a basic deref chain that terminates in a load/store/atomic
  *
  * When this pattern is found, replace the load_deref with a constant value,
  * based on which kernel argument is being loaded from. This can only be done
  * for chains that terminate in a pointer access, since the presence of null
  * pointers should be detected by actually performing the load and inspecting
  * the resulting pointer value.
  */
 static bool
 lower_deref_base_to_constant(nir_builder *b, nir_intrinsic_instr *intr, void *context)
 {
    switch (intr->intrinsic) {
    case nir_intrinsic_load_deref:
    case nir_intrinsic_store_deref:
    case nir_intrinsic_deref_atomic:
    case nir_intrinsic_deref_atomic_swap:
       break;
    default:
       return false;
    }

    nir_deref_instr *deref = nir_src_as_deref(intr->src[0]);
    if (!nir_deref_mode_must_be(deref, nir_var_mem_global | nir_var_mem_constant))
       return false;

    nir_deref_path path;
    nir_deref_path_init(&path, deref, NULL);
    bool ret = false;

    if (path.path[0]->deref_type != nir_deref_type_cast)
       goto done;
    if (!nir_deref_mode_must_be(path.path[0], nir_var_mem_global | nir_var_mem_constant))
       goto done;

    nir_instr *cast_src = nir_def_instr(path.path[0]->parent.ssa);
    if (cast_src->type != nir_instr_type_intrinsic)
       goto done;

    nir_intrinsic_instr *cast_src_intr = nir_instr_as_intrinsic(cast_src);
    if (cast_src_intr->intrinsic != nir_intrinsic_load_deref)
       goto done;

    nir_deref_instr *load_deref_src = nir_src_as_deref(cast_src_intr->src[0]);
    if (load_deref_src->deref_type != nir_deref_type_var ||
        load_deref_src->modes != nir_var_uniform)
       goto done;

    nir_variable *var = load_deref_src->var;

    ret = true;
    b->cursor = nir_before_instr(&path.path[0]->instr);
    nir_def *original_offset = nir_unpack_64_2x32_split_x(b, &cast_src_intr->def);
    nir_def *constant_ptr = nir_pack_64_2x32_split(b, original_offset, nir_imm_int(b, var->data.binding));
    nir_deref_instr *new_path = nir_build_deref_cast_with_alignment(b, constant_ptr, path.path[0]->modes, path.path[0]->type, path.path[0]->cast.ptr_stride,
                                                                    path.path[0]->cast.align_mul, path.path[0]->cast.align_offset);

    for (unsigned i = 1; path.path[i]; ++i) {
       b->cursor = nir_after_instr(&path.path[i]->instr);
       new_path = nir_build_deref_follower(b, new_path, path.path[i]);
    }
    nir_src_rewrite(&intr->src[0], &new_path->def);

 done:
    nir_deref_path_finish(&path);
    return ret;
 }

 bool
 clc_nir_lower_global_pointers_to_constants(nir_shader *nir)
 {
    return nir_shader_intrinsics_pass(nir, lower_deref_base_to_constant,
                                      nir_metadata_control_flow |
                                      nir_metadata_loop_analysis, NULL);
 }
	/*
	* Copyright © Microsoft Corporation
	*
	* Permission is hereby granted, free of charge, to any person obtaining a
	* copy of this software and associated documentation files (the "Software"),
	* to deal in the Software without restriction, including without limitation
	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	* and/or sell copies of the Software, and to permit persons to whom the
	* Software is furnished to do so, subject to the following conditions:
	*
	* The above copyright notice and this permission notice (including the next
	* paragraph) shall be included in all copies or substantial portions of the
	* Software.
	*
	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
	* IN THE SOFTWARE.
	*/

	#include "util/u_math.h"
	#include "nir.h"
	#include "glsl_types.h"
	#include "nir_builder.h"
	#include "nir_deref.h"

	#include "clc_nir.h"
	#include "clc_compiler.h"
	#include "../compiler/dxil_nir.h"

	static nir_def *
	load_ubo(nir_builder b, nir_intrinsic_instr intr, nir_variable *var, unsigned offset)
	{
	return nir_load_ubo(b,
	intr->def.num_components,
	intr->def.bit_size,
	nir_imm_int(b, var->data.binding),
	nir_imm_int(b, offset),
	.align_mul = 256,
	.align_offset = offset,
	.range_base = offset,
	.range = intr->def.bit_size * intr->def.num_components / 8);
	}

	static bool
	is_clc_system_value(const nir_instr instr, const void state)
	{
	if (instr->type != nir_instr_type_intrinsic)
	return false;
	nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
	switch (intr->intrinsic) {
	case nir_intrinsic_load_base_global_invocation_id:
	case nir_intrinsic_load_work_dim:
	case nir_intrinsic_load_num_workgroups:
	case nir_intrinsic_load_base_workgroup_id:
	return true;
	default:
	return false;
	}
	}

	static nir_def *
	lower_clc_system_value(nir_builder b, nir_instr instr, void *state)
	{
	nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
	nir_variable var = (nir_variable )state;

	switch (intr->intrinsic) {
	case nir_intrinsic_load_base_global_invocation_id:
	return load_ubo(b, intr, var, offsetof(struct clc_work_properties_data, global_offset_x));
	case nir_intrinsic_load_work_dim:
	return load_ubo(b, intr, var, offsetof(struct clc_work_properties_data, work_dim));
	case nir_intrinsic_load_num_workgroups:
	return load_ubo(b, intr, var, offsetof(struct clc_work_properties_data, group_count_total_x));
	case nir_intrinsic_load_base_workgroup_id:
	return load_ubo(b, intr, var, offsetof(struct clc_work_properties_data, group_id_offset_x));
	default:
	return NULL;
	}
	}

	bool
	clc_nir_lower_system_values(nir_shader nir, nir_variable var)
	{
	return nir_shader_lower_instructions(nir, is_clc_system_value, lower_clc_system_value, var);
	}

	static bool
	is_load_kernel_input(const nir_instr instr, const void state)
	{
	return instr->type == nir_instr_type_intrinsic &&
	nir_instr_as_intrinsic(instr)->intrinsic == nir_intrinsic_load_kernel_input;
	}

	static nir_def *
	lower_load_kernel_input(nir_builder b, nir_instr instr, void *state)
	{
	nir_variable var = (nir_variable )state;
	nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);

	unsigned bit_size = intr->def.bit_size;
	enum glsl_base_type base_type;

	switch (bit_size) {
	case 64:
	base_type = GLSL_TYPE_UINT64;
	break;
	case 32:
	base_type = GLSL_TYPE_UINT;
	break;
	case 16:
	base_type = GLSL_TYPE_UINT16;
	break;
	case 8:
	base_type = GLSL_TYPE_UINT8;
	break;
	default:
	UNREACHABLE("invalid bit size");
	}

	const struct glsl_type *type =
	glsl_vector_type(base_type, intr->def.num_components);
	nir_def *ptr = nir_vec2(b, nir_imm_int(b, var->data.binding),
	nir_u2uN(b, intr->src[0].ssa, 32));
	nir_deref_instr *deref = nir_build_deref_cast(b, ptr, nir_var_mem_ubo, type,
	bit_size / 8);
	deref->cast.align_mul = nir_intrinsic_align_mul(intr);
	deref->cast.align_offset = nir_intrinsic_align_offset(intr);

	return nir_load_deref(b, deref);
	}

	bool
	clc_nir_lower_kernel_input_loads(nir_shader nir, nir_variable var)
	{
	return nir_shader_lower_instructions(nir, is_load_kernel_input, lower_load_kernel_input, var);
	}


	static nir_variable *
	add_printf_var(struct nir_shader *nir, unsigned uav_id)
	{
	/* This size is arbitrary. Minimum required per spec is 1MB */
	const unsigned max_printf_size = 1 * 1024 * 1024;
	const unsigned printf_array_size = max_printf_size / sizeof(unsigned);
	nir_variable *var =
	nir_variable_create(nir, nir_var_mem_ssbo,
	glsl_array_type(glsl_uint_type(), printf_array_size, sizeof(unsigned)),
	"printf");
	var->data.binding = uav_id;
	return var;
	}

	bool
	clc_lower_printf_base(nir_shader *nir, unsigned uav_id)
	{
	nir_variable *printf_var = NULL;
	nir_def *printf_deref = NULL;
	nir_foreach_function_impl(impl, nir) {
	nir_builder b = nir_builder_at(nir_before_impl(impl));
	bool progress = false;

	nir_foreach_block(block, impl) {
	nir_foreach_instr_safe(instr, block) {
	if (instr->type != nir_instr_type_intrinsic)
	continue;
	nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
	if (intrin->intrinsic != nir_intrinsic_load_printf_buffer_address)
	continue;

	if (!printf_var) {
	printf_var = add_printf_var(nir, uav_id);
	nir_deref_instr *deref = nir_build_deref_var(&b, printf_var);
	printf_deref = &deref->def;
	}
	nir_def_rewrite_uses(&intrin->def, printf_deref);
	progress = true;
	}
	}

	nir_progress(progress, impl,
	nir_metadata_loop_analysis \| nir_metadata_block_index \| nir_metadata_dominance);
	}

	return printf_var != NULL;
	}

	/* Find patterns of:
	* - deref_var for one of the kernel inputs
	* - load_deref to get a pointer to global/constant memory
	* - cast the pointer into a deref
	* - use a basic deref chain that terminates in a load/store/atomic
	*
	* When this pattern is found, replace the load_deref with a constant value,
	* based on which kernel argument is being loaded from. This can only be done
	* for chains that terminate in a pointer access, since the presence of null
	* pointers should be detected by actually performing the load and inspecting
	* the resulting pointer value.
	*/
	static bool
	lower_deref_base_to_constant(nir_builder b, nir_intrinsic_instr intr, void *context)
	{
	switch (intr->intrinsic) {
	case nir_intrinsic_load_deref:
	case nir_intrinsic_store_deref:
	case nir_intrinsic_deref_atomic:
	case nir_intrinsic_deref_atomic_swap:
	break;
	default:
	return false;
	}

	nir_deref_instr *deref = nir_src_as_deref(intr->src[0]);
	if (!nir_deref_mode_must_be(deref, nir_var_mem_global \| nir_var_mem_constant))
	return false;

	nir_deref_path path;
	nir_deref_path_init(&path, deref, NULL);
	bool ret = false;

	if (path.path[0]->deref_type != nir_deref_type_cast)
	goto done;
	if (!nir_deref_mode_must_be(path.path[0], nir_var_mem_global \| nir_var_mem_constant))
	goto done;

	nir_instr *cast_src = nir_def_instr(path.path[0]->parent.ssa);
	if (cast_src->type != nir_instr_type_intrinsic)
	goto done;

	nir_intrinsic_instr *cast_src_intr = nir_instr_as_intrinsic(cast_src);
	if (cast_src_intr->intrinsic != nir_intrinsic_load_deref)
	goto done;

	nir_deref_instr *load_deref_src = nir_src_as_deref(cast_src_intr->src[0]);
	if (load_deref_src->deref_type != nir_deref_type_var \|\|
	load_deref_src->modes != nir_var_uniform)
	goto done;

	nir_variable *var = load_deref_src->var;

	ret = true;
	b->cursor = nir_before_instr(&path.path[0]->instr);
	nir_def *original_offset = nir_unpack_64_2x32_split_x(b, &cast_src_intr->def);
	nir_def *constant_ptr = nir_pack_64_2x32_split(b, original_offset, nir_imm_int(b, var->data.binding));
	nir_deref_instr *new_path = nir_build_deref_cast_with_alignment(b, constant_ptr, path.path[0]->modes, path.path[0]->type, path.path[0]->cast.ptr_stride,
	path.path[0]->cast.align_mul, path.path[0]->cast.align_offset);

	for (unsigned i = 1; path.path[i]; ++i) {
	b->cursor = nir_after_instr(&path.path[i]->instr);
	new_path = nir_build_deref_follower(b, new_path, path.path[i]);
	}
	nir_src_rewrite(&intr->src[0], &new_path->def);

	done:
	nir_deref_path_finish(&path);
	return ret;
	}

	bool
	clc_nir_lower_global_pointers_to_constants(nir_shader *nir)
	{
	return nir_shader_intrinsics_pass(nir, lower_deref_base_to_constant,
	nir_metadata_control_flow \|
	nir_metadata_loop_analysis, NULL);
	}