| /* |
| * Copyright © 2018 Valve Corporation |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice (including the next |
| * paragraph) shall be included in all copies or substantial portions of the |
| * Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
| * IN THE SOFTWARE. |
| * |
| */ |
| |
| #include "nir.h" |
| |
| /* This pass computes for each ssa definition if it is uniform. |
| * That is, the variable has the same value for all invocations |
| * of the group. |
| * |
| * This divergence analysis pass expects the shader to be in LCSSA-form. |
| * |
| * This algorithm implements "The Simple Divergence Analysis" from |
| * Diogo Sampaio, Rafael De Souza, Sylvain Collange, Fernando Magno Quintão Pereira. |
| * Divergence Analysis. ACM Transactions on Programming Languages and Systems (TOPLAS), |
| * ACM, 2013, 35 (4), pp.13:1-13:36. <10.1145/2523815>. <hal-00909072v2> |
| */ |
| |
| static bool |
| visit_cf_list(bool *divergent, struct exec_list *list, |
| nir_divergence_options options, gl_shader_stage stage); |
| |
| static bool |
| visit_alu(bool *divergent, nir_alu_instr *instr) |
| { |
| if (divergent[instr->dest.dest.ssa.index]) |
| return false; |
| |
| unsigned num_src = nir_op_infos[instr->op].num_inputs; |
| |
| for (unsigned i = 0; i < num_src; i++) { |
| if (divergent[instr->src[i].src.ssa->index]) { |
| divergent[instr->dest.dest.ssa.index] = true; |
| return true; |
| } |
| } |
| |
| return false; |
| } |
| |
| static bool |
| visit_intrinsic(bool *divergent, nir_intrinsic_instr *instr, |
| nir_divergence_options options, gl_shader_stage stage) |
| { |
| if (!nir_intrinsic_infos[instr->intrinsic].has_dest) |
| return false; |
| |
| if (divergent[instr->dest.ssa.index]) |
| return false; |
| |
| bool is_divergent = false; |
| switch (instr->intrinsic) { |
| /* Intrinsics which are always uniform */ |
| case nir_intrinsic_shader_clock: |
| case nir_intrinsic_ballot: |
| case nir_intrinsic_read_invocation: |
| case nir_intrinsic_read_first_invocation: |
| case nir_intrinsic_vote_any: |
| case nir_intrinsic_vote_all: |
| case nir_intrinsic_vote_feq: |
| case nir_intrinsic_vote_ieq: |
| case nir_intrinsic_load_work_dim: |
| case nir_intrinsic_load_work_group_id: |
| case nir_intrinsic_load_num_work_groups: |
| case nir_intrinsic_load_local_group_size: |
| case nir_intrinsic_load_subgroup_id: |
| case nir_intrinsic_load_num_subgroups: |
| case nir_intrinsic_load_subgroup_size: |
| case nir_intrinsic_load_subgroup_eq_mask: |
| case nir_intrinsic_load_subgroup_ge_mask: |
| case nir_intrinsic_load_subgroup_gt_mask: |
| case nir_intrinsic_load_subgroup_le_mask: |
| case nir_intrinsic_load_subgroup_lt_mask: |
| case nir_intrinsic_first_invocation: |
| case nir_intrinsic_load_base_instance: |
| case nir_intrinsic_load_base_vertex: |
| case nir_intrinsic_load_first_vertex: |
| case nir_intrinsic_load_draw_id: |
| case nir_intrinsic_load_is_indexed_draw: |
| case nir_intrinsic_load_viewport_scale: |
| case nir_intrinsic_load_alpha_ref_float: |
| case nir_intrinsic_load_user_clip_plane: |
| case nir_intrinsic_load_viewport_x_scale: |
| case nir_intrinsic_load_viewport_y_scale: |
| case nir_intrinsic_load_viewport_z_scale: |
| case nir_intrinsic_load_viewport_offset: |
| case nir_intrinsic_load_viewport_z_offset: |
| case nir_intrinsic_load_blend_const_color_a_float: |
| case nir_intrinsic_load_blend_const_color_b_float: |
| case nir_intrinsic_load_blend_const_color_g_float: |
| case nir_intrinsic_load_blend_const_color_r_float: |
| case nir_intrinsic_load_blend_const_color_rgba: |
| case nir_intrinsic_load_blend_const_color_aaaa8888_unorm: |
| case nir_intrinsic_load_blend_const_color_rgba8888_unorm: |
| is_divergent = false; |
| break; |
| |
| /* Intrinsics with divergence depending on shader stage and hardware */ |
| case nir_intrinsic_load_input: |
| is_divergent = divergent[instr->src[0].ssa->index]; |
| if (stage == MESA_SHADER_FRAGMENT) |
| is_divergent |= !(options & nir_divergence_single_prim_per_subgroup); |
| else if (stage == MESA_SHADER_TESS_EVAL) |
| is_divergent |= !(options & nir_divergence_single_patch_per_tes_subgroup); |
| else |
| is_divergent = true; |
| break; |
| case nir_intrinsic_load_output: |
| assert(stage == MESA_SHADER_TESS_CTRL || stage == MESA_SHADER_FRAGMENT); |
| is_divergent = divergent[instr->src[0].ssa->index]; |
| if (stage == MESA_SHADER_TESS_CTRL) |
| is_divergent |= !(options & nir_divergence_single_patch_per_tcs_subgroup); |
| else |
| is_divergent = true; |
| break; |
| case nir_intrinsic_load_layer_id: |
| case nir_intrinsic_load_front_face: |
| assert(stage == MESA_SHADER_FRAGMENT); |
| is_divergent = !(options & nir_divergence_single_prim_per_subgroup); |
| break; |
| case nir_intrinsic_load_view_index: |
| assert(stage != MESA_SHADER_COMPUTE && stage != MESA_SHADER_KERNEL); |
| if (options & nir_divergence_view_index_uniform) |
| is_divergent = false; |
| else if (stage == MESA_SHADER_FRAGMENT) |
| is_divergent = !(options & nir_divergence_single_prim_per_subgroup); |
| break; |
| case nir_intrinsic_load_fs_input_interp_deltas: |
| assert(stage == MESA_SHADER_FRAGMENT); |
| is_divergent = divergent[instr->src[0].ssa->index]; |
| is_divergent |= !(options & nir_divergence_single_prim_per_subgroup); |
| break; |
| case nir_intrinsic_load_primitive_id: |
| if (stage == MESA_SHADER_FRAGMENT) |
| is_divergent = !(options & nir_divergence_single_prim_per_subgroup); |
| else if (stage == MESA_SHADER_TESS_CTRL) |
| is_divergent = !(options & nir_divergence_single_patch_per_tcs_subgroup); |
| else if (stage == MESA_SHADER_TESS_EVAL) |
| is_divergent = !(options & nir_divergence_single_patch_per_tes_subgroup); |
| else |
| unreachable("Invalid stage for load_primitive_id"); |
| break; |
| case nir_intrinsic_load_tess_level_inner: |
| case nir_intrinsic_load_tess_level_outer: |
| if (stage == MESA_SHADER_TESS_CTRL) |
| is_divergent = !(options & nir_divergence_single_patch_per_tcs_subgroup); |
| else if (stage == MESA_SHADER_TESS_EVAL) |
| is_divergent = !(options & nir_divergence_single_patch_per_tes_subgroup); |
| else |
| unreachable("Invalid stage for load_primitive_tess_level_*"); |
| break; |
| case nir_intrinsic_load_patch_vertices_in: |
| if (stage == MESA_SHADER_TESS_EVAL) |
| is_divergent = !(options & nir_divergence_single_patch_per_tes_subgroup); |
| else |
| assert(stage == MESA_SHADER_TESS_CTRL); |
| break; |
| |
| /* Clustered reductions are uniform if cluster_size == subgroup_size or |
| * the source is uniform and the operation is invariant. |
| * Inclusive scans are uniform if |
| * the source is uniform and the operation is invariant |
| */ |
| case nir_intrinsic_reduce: |
| if (nir_intrinsic_cluster_size(instr) == 0) |
| return false; |
| /* fallthrough */ |
| case nir_intrinsic_inclusive_scan: { |
| nir_op op = nir_intrinsic_reduction_op(instr); |
| is_divergent = divergent[instr->src[0].ssa->index]; |
| if (op != nir_op_umin && op != nir_op_imin && op != nir_op_fmin && |
| op != nir_op_umax && op != nir_op_imax && op != nir_op_fmax && |
| op != nir_op_iand && op != nir_op_ior) |
| is_divergent = true; |
| break; |
| } |
| |
| /* Intrinsics with divergence depending on sources */ |
| case nir_intrinsic_ballot_bitfield_extract: |
| case nir_intrinsic_ballot_find_lsb: |
| case nir_intrinsic_ballot_find_msb: |
| case nir_intrinsic_ballot_bit_count_reduce: |
| case nir_intrinsic_shuffle: |
| case nir_intrinsic_shuffle_xor: |
| case nir_intrinsic_shuffle_up: |
| case nir_intrinsic_shuffle_down: |
| case nir_intrinsic_quad_broadcast: |
| case nir_intrinsic_quad_swap_horizontal: |
| case nir_intrinsic_quad_swap_vertical: |
| case nir_intrinsic_quad_swap_diagonal: |
| case nir_intrinsic_load_deref: |
| case nir_intrinsic_load_ubo: |
| case nir_intrinsic_load_ssbo: |
| case nir_intrinsic_load_shared: |
| case nir_intrinsic_load_global: |
| case nir_intrinsic_load_uniform: |
| case nir_intrinsic_load_push_constant: |
| case nir_intrinsic_load_constant: |
| case nir_intrinsic_load_sample_pos_from_id: |
| case nir_intrinsic_load_kernel_input: |
| case nir_intrinsic_image_load: |
| case nir_intrinsic_image_deref_load: |
| case nir_intrinsic_bindless_image_load: |
| case nir_intrinsic_image_samples: |
| case nir_intrinsic_image_deref_samples: |
| case nir_intrinsic_bindless_image_samples: |
| case nir_intrinsic_get_buffer_size: |
| case nir_intrinsic_image_size: |
| case nir_intrinsic_image_deref_size: |
| case nir_intrinsic_bindless_image_size: |
| case nir_intrinsic_copy_deref: |
| case nir_intrinsic_deref_buffer_array_length: |
| case nir_intrinsic_vulkan_resource_index: |
| case nir_intrinsic_vulkan_resource_reindex: |
| case nir_intrinsic_load_vulkan_descriptor: |
| case nir_intrinsic_atomic_counter_read: |
| case nir_intrinsic_atomic_counter_read_deref: |
| case nir_intrinsic_quad_swizzle_amd: |
| case nir_intrinsic_masked_swizzle_amd: { |
| unsigned num_srcs = nir_intrinsic_infos[instr->intrinsic].num_srcs; |
| for (unsigned i = 0; i < num_srcs; i++) { |
| if (divergent[instr->src[i].ssa->index]) { |
| is_divergent = true; |
| break; |
| } |
| } |
| break; |
| } |
| |
| /* Intrinsics which are always divergent */ |
| case nir_intrinsic_load_color0: |
| case nir_intrinsic_load_color1: |
| case nir_intrinsic_load_param: |
| case nir_intrinsic_load_sample_id: |
| case nir_intrinsic_load_sample_id_no_per_sample: |
| case nir_intrinsic_load_sample_mask_in: |
| case nir_intrinsic_load_interpolated_input: |
| case nir_intrinsic_load_barycentric_pixel: |
| case nir_intrinsic_load_barycentric_centroid: |
| case nir_intrinsic_load_barycentric_sample: |
| case nir_intrinsic_load_barycentric_at_sample: |
| case nir_intrinsic_load_barycentric_at_offset: |
| case nir_intrinsic_interp_deref_at_offset: |
| case nir_intrinsic_interp_deref_at_sample: |
| case nir_intrinsic_interp_deref_at_centroid: |
| case nir_intrinsic_load_tess_coord: |
| case nir_intrinsic_load_point_coord: |
| case nir_intrinsic_load_frag_coord: |
| case nir_intrinsic_load_sample_pos: |
| case nir_intrinsic_load_vertex_id_zero_base: |
| case nir_intrinsic_load_vertex_id: |
| case nir_intrinsic_load_per_vertex_input: |
| case nir_intrinsic_load_per_vertex_output: |
| case nir_intrinsic_load_instance_id: |
| case nir_intrinsic_load_invocation_id: |
| case nir_intrinsic_load_local_invocation_id: |
| case nir_intrinsic_load_local_invocation_index: |
| case nir_intrinsic_load_global_invocation_id: |
| case nir_intrinsic_load_global_invocation_index: |
| case nir_intrinsic_load_subgroup_invocation: |
| case nir_intrinsic_load_helper_invocation: |
| case nir_intrinsic_is_helper_invocation: |
| case nir_intrinsic_load_scratch: |
| case nir_intrinsic_deref_atomic_add: |
| case nir_intrinsic_deref_atomic_imin: |
| case nir_intrinsic_deref_atomic_umin: |
| case nir_intrinsic_deref_atomic_imax: |
| case nir_intrinsic_deref_atomic_umax: |
| case nir_intrinsic_deref_atomic_and: |
| case nir_intrinsic_deref_atomic_or: |
| case nir_intrinsic_deref_atomic_xor: |
| case nir_intrinsic_deref_atomic_exchange: |
| case nir_intrinsic_deref_atomic_comp_swap: |
| case nir_intrinsic_deref_atomic_fadd: |
| case nir_intrinsic_deref_atomic_fmin: |
| case nir_intrinsic_deref_atomic_fmax: |
| case nir_intrinsic_deref_atomic_fcomp_swap: |
| case nir_intrinsic_ssbo_atomic_add: |
| case nir_intrinsic_ssbo_atomic_imin: |
| case nir_intrinsic_ssbo_atomic_umin: |
| case nir_intrinsic_ssbo_atomic_imax: |
| case nir_intrinsic_ssbo_atomic_umax: |
| case nir_intrinsic_ssbo_atomic_and: |
| case nir_intrinsic_ssbo_atomic_or: |
| case nir_intrinsic_ssbo_atomic_xor: |
| case nir_intrinsic_ssbo_atomic_exchange: |
| case nir_intrinsic_ssbo_atomic_comp_swap: |
| case nir_intrinsic_ssbo_atomic_fadd: |
| case nir_intrinsic_ssbo_atomic_fmax: |
| case nir_intrinsic_ssbo_atomic_fmin: |
| case nir_intrinsic_ssbo_atomic_fcomp_swap: |
| case nir_intrinsic_image_deref_atomic_add: |
| case nir_intrinsic_image_deref_atomic_min: |
| case nir_intrinsic_image_deref_atomic_max: |
| case nir_intrinsic_image_deref_atomic_and: |
| case nir_intrinsic_image_deref_atomic_or: |
| case nir_intrinsic_image_deref_atomic_xor: |
| case nir_intrinsic_image_deref_atomic_exchange: |
| case nir_intrinsic_image_deref_atomic_comp_swap: |
| case nir_intrinsic_image_deref_atomic_fadd: |
| case nir_intrinsic_image_atomic_add: |
| case nir_intrinsic_image_atomic_min: |
| case nir_intrinsic_image_atomic_max: |
| case nir_intrinsic_image_atomic_and: |
| case nir_intrinsic_image_atomic_or: |
| case nir_intrinsic_image_atomic_xor: |
| case nir_intrinsic_image_atomic_exchange: |
| case nir_intrinsic_image_atomic_comp_swap: |
| case nir_intrinsic_image_atomic_fadd: |
| case nir_intrinsic_bindless_image_atomic_add: |
| case nir_intrinsic_bindless_image_atomic_min: |
| case nir_intrinsic_bindless_image_atomic_max: |
| case nir_intrinsic_bindless_image_atomic_and: |
| case nir_intrinsic_bindless_image_atomic_or: |
| case nir_intrinsic_bindless_image_atomic_xor: |
| case nir_intrinsic_bindless_image_atomic_exchange: |
| case nir_intrinsic_bindless_image_atomic_comp_swap: |
| case nir_intrinsic_bindless_image_atomic_fadd: |
| case nir_intrinsic_shared_atomic_add: |
| case nir_intrinsic_shared_atomic_imin: |
| case nir_intrinsic_shared_atomic_umin: |
| case nir_intrinsic_shared_atomic_imax: |
| case nir_intrinsic_shared_atomic_umax: |
| case nir_intrinsic_shared_atomic_and: |
| case nir_intrinsic_shared_atomic_or: |
| case nir_intrinsic_shared_atomic_xor: |
| case nir_intrinsic_shared_atomic_exchange: |
| case nir_intrinsic_shared_atomic_comp_swap: |
| case nir_intrinsic_shared_atomic_fadd: |
| case nir_intrinsic_shared_atomic_fmin: |
| case nir_intrinsic_shared_atomic_fmax: |
| case nir_intrinsic_shared_atomic_fcomp_swap: |
| case nir_intrinsic_global_atomic_add: |
| case nir_intrinsic_global_atomic_imin: |
| case nir_intrinsic_global_atomic_umin: |
| case nir_intrinsic_global_atomic_imax: |
| case nir_intrinsic_global_atomic_umax: |
| case nir_intrinsic_global_atomic_and: |
| case nir_intrinsic_global_atomic_or: |
| case nir_intrinsic_global_atomic_xor: |
| case nir_intrinsic_global_atomic_exchange: |
| case nir_intrinsic_global_atomic_comp_swap: |
| case nir_intrinsic_global_atomic_fadd: |
| case nir_intrinsic_global_atomic_fmin: |
| case nir_intrinsic_global_atomic_fmax: |
| case nir_intrinsic_global_atomic_fcomp_swap: |
| case nir_intrinsic_atomic_counter_add: |
| case nir_intrinsic_atomic_counter_min: |
| case nir_intrinsic_atomic_counter_max: |
| case nir_intrinsic_atomic_counter_and: |
| case nir_intrinsic_atomic_counter_or: |
| case nir_intrinsic_atomic_counter_xor: |
| case nir_intrinsic_atomic_counter_inc: |
| case nir_intrinsic_atomic_counter_pre_dec: |
| case nir_intrinsic_atomic_counter_post_dec: |
| case nir_intrinsic_atomic_counter_exchange: |
| case nir_intrinsic_atomic_counter_comp_swap: |
| case nir_intrinsic_atomic_counter_add_deref: |
| case nir_intrinsic_atomic_counter_min_deref: |
| case nir_intrinsic_atomic_counter_max_deref: |
| case nir_intrinsic_atomic_counter_and_deref: |
| case nir_intrinsic_atomic_counter_or_deref: |
| case nir_intrinsic_atomic_counter_xor_deref: |
| case nir_intrinsic_atomic_counter_inc_deref: |
| case nir_intrinsic_atomic_counter_pre_dec_deref: |
| case nir_intrinsic_atomic_counter_post_dec_deref: |
| case nir_intrinsic_atomic_counter_exchange_deref: |
| case nir_intrinsic_atomic_counter_comp_swap_deref: |
| case nir_intrinsic_exclusive_scan: |
| case nir_intrinsic_ballot_bit_count_exclusive: |
| case nir_intrinsic_ballot_bit_count_inclusive: |
| case nir_intrinsic_write_invocation_amd: |
| case nir_intrinsic_mbcnt_amd: |
| is_divergent = true; |
| break; |
| |
| default: |
| #ifdef NDEBUG |
| is_divergent = true; |
| break; |
| #else |
| nir_print_instr(&instr->instr, stderr); |
| unreachable("\nNIR divergence analysis: Unhandled intrinsic."); |
| #endif |
| } |
| |
| divergent[instr->dest.ssa.index] = is_divergent; |
| return is_divergent; |
| } |
| |
| static bool |
| visit_tex(bool *divergent, nir_tex_instr *instr) |
| { |
| if (divergent[instr->dest.ssa.index]) |
| return false; |
| |
| bool is_divergent = false; |
| |
| for (unsigned i = 0; i < instr->num_srcs; i++) { |
| switch (instr->src[i].src_type) { |
| case nir_tex_src_sampler_deref: |
| case nir_tex_src_sampler_handle: |
| case nir_tex_src_sampler_offset: |
| is_divergent |= divergent[instr->src[i].src.ssa->index] && |
| instr->sampler_non_uniform; |
| break; |
| case nir_tex_src_texture_deref: |
| case nir_tex_src_texture_handle: |
| case nir_tex_src_texture_offset: |
| is_divergent |= divergent[instr->src[i].src.ssa->index] && |
| instr->texture_non_uniform; |
| break; |
| default: |
| is_divergent |= divergent[instr->src[i].src.ssa->index]; |
| break; |
| } |
| } |
| |
| divergent[instr->dest.ssa.index] = is_divergent; |
| return is_divergent; |
| } |
| |
| static bool |
| visit_phi(bool *divergent, nir_phi_instr *instr) |
| { |
| /* There are 3 types of phi instructions: |
| * (1) gamma: represent the joining point of different paths |
| * created by an “if-then-else” branch. |
| * The resulting value is divergent if the branch condition |
| * or any of the source values is divergent. |
| * |
| * (2) mu: which only exist at loop headers, |
| * merge initial and loop-carried values. |
| * The resulting value is divergent if any source value |
| * is divergent or a divergent loop continue condition |
| * is associated with a different ssa-def. |
| * |
| * (3) eta: represent values that leave a loop. |
| * The resulting value is divergent if the source value is divergent |
| * or any loop exit condition is divergent for a value which is |
| * not loop-invariant. |
| * (note: there should be no phi for loop-invariant variables.) |
| */ |
| |
| if (divergent[instr->dest.ssa.index]) |
| return false; |
| |
| nir_foreach_phi_src(src, instr) { |
| /* if any source value is divergent, the resulting value is divergent */ |
| if (divergent[src->src.ssa->index]) { |
| divergent[instr->dest.ssa.index] = true; |
| return true; |
| } |
| } |
| |
| nir_cf_node *prev = nir_cf_node_prev(&instr->instr.block->cf_node); |
| |
| if (!prev) { |
| /* mu: if no predecessor node exists, the phi must be at a loop header */ |
| nir_loop *loop = nir_cf_node_as_loop(instr->instr.block->cf_node.parent); |
| prev = nir_cf_node_prev(&loop->cf_node); |
| nir_ssa_def* same = NULL; |
| bool all_same = true; |
| |
| /* first, check if all loop-carried values are from the same ssa-def */ |
| nir_foreach_phi_src(src, instr) { |
| if (src->pred == nir_cf_node_as_block(prev)) |
| continue; |
| if (src->src.ssa->parent_instr->type == nir_instr_type_ssa_undef) |
| continue; |
| if (!same) |
| same = src->src.ssa; |
| else if (same != src->src.ssa) |
| all_same = false; |
| } |
| |
| /* if all loop-carried values are the same, the resulting value is uniform */ |
| if (all_same) |
| return false; |
| |
| /* check if the loop-carried values come from different ssa-defs |
| * and the corresponding condition is divergent. */ |
| nir_foreach_phi_src(src, instr) { |
| /* skip the loop preheader */ |
| if (src->pred == nir_cf_node_as_block(prev)) |
| continue; |
| |
| /* skip the unconditional back-edge */ |
| if (src->pred == nir_loop_last_block(loop)) |
| continue; |
| |
| /* if the value is undef, we don't need to check the condition */ |
| if (src->src.ssa->parent_instr->type == nir_instr_type_ssa_undef) |
| continue; |
| |
| nir_cf_node *current = src->pred->cf_node.parent; |
| /* check recursively the conditions if any is divergent */ |
| while (current->type != nir_cf_node_loop) { |
| assert (current->type == nir_cf_node_if); |
| nir_if *if_node = nir_cf_node_as_if(current); |
| if (divergent[if_node->condition.ssa->index]) { |
| divergent[instr->dest.ssa.index] = true; |
| return true; |
| } |
| current = current->parent; |
| } |
| assert(current == &loop->cf_node); |
| } |
| |
| } else if (prev->type == nir_cf_node_if) { |
| /* if only one of the incoming values is defined, the resulting value is uniform */ |
| unsigned defined_srcs = 0; |
| nir_foreach_phi_src(src, instr) { |
| if (src->src.ssa->parent_instr->type != nir_instr_type_ssa_undef) |
| defined_srcs++; |
| } |
| if (defined_srcs <= 1) |
| return false; |
| |
| /* gamma: check if the condition is divergent */ |
| nir_if *if_node = nir_cf_node_as_if(prev); |
| if (divergent[if_node->condition.ssa->index]) { |
| divergent[instr->dest.ssa.index] = true; |
| return true; |
| } |
| |
| } else { |
| /* eta: the predecessor must be a loop */ |
| assert(prev->type == nir_cf_node_loop); |
| |
| /* Check if any loop exit condition is divergent: |
| * That is any break happens under divergent condition or |
| * a break is preceeded by a divergent continue |
| */ |
| nir_foreach_phi_src(src, instr) { |
| nir_cf_node *current = src->pred->cf_node.parent; |
| |
| /* check recursively the conditions if any is divergent */ |
| while (current->type != nir_cf_node_loop) { |
| assert(current->type == nir_cf_node_if); |
| nir_if *if_node = nir_cf_node_as_if(current); |
| if (divergent[if_node->condition.ssa->index]) { |
| divergent[instr->dest.ssa.index] = true; |
| return true; |
| } |
| current = current->parent; |
| } |
| assert(current == prev); |
| |
| /* check if any divergent continue happened before the break */ |
| nir_foreach_block_in_cf_node(block, prev) { |
| if (block == src->pred) |
| break; |
| if (!nir_block_ends_in_jump(block)) |
| continue; |
| |
| nir_jump_instr *jump = nir_instr_as_jump(nir_block_last_instr(block)); |
| if (jump->type != nir_jump_continue) |
| continue; |
| |
| current = block->cf_node.parent; |
| bool is_divergent = false; |
| while (current != prev) { |
| /* the continue belongs to an inner loop */ |
| if (current->type == nir_cf_node_loop) { |
| is_divergent = false; |
| break; |
| } |
| assert(current->type == nir_cf_node_if); |
| nir_if *if_node = nir_cf_node_as_if(current); |
| is_divergent |= divergent[if_node->condition.ssa->index]; |
| current = current->parent; |
| } |
| |
| if (is_divergent) { |
| divergent[instr->dest.ssa.index] = true; |
| return true; |
| } |
| } |
| } |
| } |
| |
| return false; |
| } |
| |
| static bool |
| visit_load_const(bool *divergent, nir_load_const_instr *instr) |
| { |
| return false; |
| } |
| |
| static bool |
| visit_ssa_undef(bool *divergent, nir_ssa_undef_instr *instr) |
| { |
| return false; |
| } |
| |
| static bool |
| nir_variable_mode_is_uniform(nir_variable_mode mode) { |
| switch (mode) { |
| case nir_var_uniform: |
| case nir_var_mem_ubo: |
| case nir_var_mem_ssbo: |
| case nir_var_mem_shared: |
| case nir_var_mem_global: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| static bool |
| nir_variable_is_uniform(nir_variable *var, nir_divergence_options options, |
| gl_shader_stage stage) |
| { |
| if (nir_variable_mode_is_uniform(var->data.mode)) |
| return true; |
| |
| if (stage == MESA_SHADER_FRAGMENT && |
| (options & nir_divergence_single_prim_per_subgroup) && |
| var->data.mode == nir_var_shader_in && |
| var->data.interpolation == INTERP_MODE_FLAT) |
| return true; |
| |
| if (stage == MESA_SHADER_TESS_CTRL && |
| (options & nir_divergence_single_patch_per_tcs_subgroup) && |
| var->data.mode == nir_var_shader_out && var->data.patch) |
| return true; |
| |
| if (stage == MESA_SHADER_TESS_EVAL && |
| (options & nir_divergence_single_patch_per_tes_subgroup) && |
| var->data.mode == nir_var_shader_in && var->data.patch) |
| return true; |
| |
| return false; |
| } |
| |
| static bool |
| visit_deref(bool *divergent, nir_deref_instr *deref, |
| nir_divergence_options options, gl_shader_stage stage) |
| { |
| if (divergent[deref->dest.ssa.index]) |
| return false; |
| |
| bool is_divergent = false; |
| switch (deref->deref_type) { |
| case nir_deref_type_var: |
| is_divergent = !nir_variable_is_uniform(deref->var, options, stage); |
| break; |
| case nir_deref_type_array: |
| case nir_deref_type_ptr_as_array: |
| is_divergent = divergent[deref->arr.index.ssa->index]; |
| /* fallthrough */ |
| case nir_deref_type_struct: |
| case nir_deref_type_array_wildcard: |
| is_divergent |= divergent[deref->parent.ssa->index]; |
| break; |
| case nir_deref_type_cast: |
| is_divergent = !nir_variable_mode_is_uniform(deref->var->data.mode) || |
| divergent[deref->parent.ssa->index]; |
| break; |
| } |
| |
| divergent[deref->dest.ssa.index] = is_divergent; |
| return is_divergent; |
| } |
| |
| static bool |
| visit_block(bool *divergent, nir_block *block, nir_divergence_options options, |
| gl_shader_stage stage) |
| { |
| bool has_changed = false; |
| |
| nir_foreach_instr(instr, block) { |
| switch (instr->type) { |
| case nir_instr_type_alu: |
| has_changed |= visit_alu(divergent, nir_instr_as_alu(instr)); |
| break; |
| case nir_instr_type_intrinsic: |
| has_changed |= visit_intrinsic(divergent, nir_instr_as_intrinsic(instr), |
| options, stage); |
| break; |
| case nir_instr_type_tex: |
| has_changed |= visit_tex(divergent, nir_instr_as_tex(instr)); |
| break; |
| case nir_instr_type_phi: |
| has_changed |= visit_phi(divergent, nir_instr_as_phi(instr)); |
| break; |
| case nir_instr_type_load_const: |
| has_changed |= visit_load_const(divergent, nir_instr_as_load_const(instr)); |
| break; |
| case nir_instr_type_ssa_undef: |
| has_changed |= visit_ssa_undef(divergent, nir_instr_as_ssa_undef(instr)); |
| break; |
| case nir_instr_type_deref: |
| has_changed |= visit_deref(divergent, nir_instr_as_deref(instr), |
| options, stage); |
| break; |
| case nir_instr_type_jump: |
| break; |
| case nir_instr_type_call: |
| case nir_instr_type_parallel_copy: |
| unreachable("NIR divergence analysis: Unsupported instruction type."); |
| } |
| } |
| |
| return has_changed; |
| } |
| |
| static bool |
| visit_if(bool *divergent, nir_if *if_stmt, nir_divergence_options options, gl_shader_stage stage) |
| { |
| return visit_cf_list(divergent, &if_stmt->then_list, options, stage) | |
| visit_cf_list(divergent, &if_stmt->else_list, options, stage); |
| } |
| |
| static bool |
| visit_loop(bool *divergent, nir_loop *loop, nir_divergence_options options, gl_shader_stage stage) |
| { |
| bool has_changed = false; |
| bool repeat = true; |
| |
| /* TODO: restructure this and the phi handling more efficiently */ |
| while (repeat) { |
| repeat = visit_cf_list(divergent, &loop->body, options, stage); |
| has_changed |= repeat; |
| } |
| |
| return has_changed; |
| } |
| |
| static bool |
| visit_cf_list(bool *divergent, struct exec_list *list, |
| nir_divergence_options options, gl_shader_stage stage) |
| { |
| bool has_changed = false; |
| |
| foreach_list_typed(nir_cf_node, node, node, list) { |
| switch (node->type) { |
| case nir_cf_node_block: |
| has_changed |= visit_block(divergent, nir_cf_node_as_block(node), |
| options, stage); |
| break; |
| case nir_cf_node_if: |
| has_changed |= visit_if(divergent, nir_cf_node_as_if(node), |
| options, stage); |
| break; |
| case nir_cf_node_loop: |
| has_changed |= visit_loop(divergent, nir_cf_node_as_loop(node), |
| options, stage); |
| break; |
| case nir_cf_node_function: |
| unreachable("NIR divergence analysis: Unsupported cf_node type."); |
| } |
| } |
| |
| return has_changed; |
| } |
| |
| |
| bool* |
| nir_divergence_analysis(nir_shader *shader, nir_divergence_options options) |
| { |
| nir_function_impl *impl = nir_shader_get_entrypoint(shader); |
| bool *t = rzalloc_array(shader, bool, impl->ssa_alloc); |
| |
| visit_cf_list(t, &impl->body, options, shader->info.stage); |
| |
| return t; |
| } |
