| /* Copyright © 2022 Friedrich Vock |
| * Copyright © 2024 Intel Corporation |
| * SPDX-License-Identifier: MIT |
| */ |
| |
| #version 460 |
| |
| layout(local_size_x = 32, local_size_y = 1, local_size_z = 1) in; |
| |
| #include "anv_bvh_defines.h" |
| #define VK_USED_BUILD_FLAGS ANV_ENCODE_BUILD_FLAGS |
| #include "anv_bvh_helpers.h" |
| #include "encode.h" |
| |
| #define READY_TO_WRITE(offset) ((offset) < VK_NULL_BVH_OFFSET) |
| #define ASSIGNED_NODE_TO_ENCODE ((gl_GlobalInvocationID.x + push_args.start_node_offset) \ |
| < DEREF(args.header).ir_internal_node_count) |
| |
| /* Debugging helper: disable encoding by exiting early. Ensure compiler doesn't |
| * dead code eliminate by comparing to value that should never evaluate as true. |
| */ |
| #define DEBUG_DISABLE_WRITE 0 |
| #define DEBUG_EXIT_EARLY(val) (DEBUG_DISABLE_WRITE == 1) && ((val) != 123456) |
| |
| /* IR_NODE refers to memory that holds IR NODEs which are to be encoded. */ |
| #define IR_NODE uint32_t |
| #define NODE_OFFSET(node) (OFFSET(args.intermediate_bvh, ir_id_to_offset(node))) |
| |
| layout(push_constant) uniform CONSTS { |
| encode_args push_args; |
| }; |
| anv_batch_args args = DEREF(INDEX(anv_batch_args, push_args.batch_args, |
| push_args.batch_offset + gl_GlobalInvocationID.y)); |
| |
| void |
| debug_dump(uint32_t offset, uint32_t value) |
| { |
| REF(uint32_t) msg = REF(uint32_t)(OFFSET(args.output_bvh, offset)); |
| DEREF(msg) = value; |
| } |
| |
| struct anv_cluster { |
| /* simd lane inside cluster: 0 .. 7 */ |
| uint32_t idx; |
| |
| /* ID of cluster: 0 .. globalInvocations.x/8-1 */ |
| uint32_t cluster_id; |
| |
| /* size = 8 */ |
| uint32_t size; |
| }; |
| |
| /* cluster_size has to be a power of two and <32. */ |
| void |
| anv_cluster_init(out anv_cluster cluster, uint32_t size) |
| { |
| cluster.idx = gl_SubgroupInvocationID & (size - 1); |
| cluster.cluster_id = gl_SubgroupInvocationID / size; |
| cluster.size = size; |
| } |
| |
| #define anv_shuffle(cluster, cluster_idx, value) \ |
| subgroupShuffle(value, (gl_SubgroupInvocationID & (~(cluster.size - 1))) + cluster_idx) |
| |
| void |
| encode_leaf_node(uint32_t type, IR_NODE child, uint64_t dst_node, REF(anv_accel_struct_header) dst_header) |
| { |
| uint64_t src_node = NODE_OFFSET(child); |
| |
| if (DEBUG_EXIT_EARLY(type)) |
| return; |
| |
| switch (type) { |
| case vk_ir_node_triangle: { |
| vk_ir_triangle_node src = DEREF(REF(vk_ir_triangle_node)(src_node)); |
| |
| if (VK_TEST_BUILD_FLAG_HAS_QUADS) { |
| vk_ir_triangle_node_quad quad = |
| vk_ir_triangle_node_get_quad(REF(vk_ir_triangle_node)(src_node)); |
| if (quad.triangle_id == VK_QUAD_TRIANGLE_ID_UNUSED) { |
| anv_encode_triangle(dst_node, src); |
| } else { |
| anv_encode_quad(dst_node, src, quad); |
| } |
| break; |
| } |
| |
| anv_encode_triangle(dst_node, src); |
| break; |
| } |
| case vk_ir_node_aabb: { |
| vk_ir_aabb_node src = DEREF(REF(vk_ir_aabb_node)(src_node)); |
| anv_encode_aabb(dst_node, src); |
| break; |
| } |
| case vk_ir_node_instance: { |
| vk_ir_instance_node src = DEREF(REF(vk_ir_instance_node)(src_node)); |
| anv_encode_instance(dst_node, src); |
| uint64_t instance_leaves_addr_base = args.instance_leaves_addr; |
| uint64_t slot = ir_id_to_offset(child) / SIZEOF(vk_ir_instance_node); |
| DEREF(INDEX(uint64_t, instance_leaves_addr_base, slot)) = dst_node; |
| break; |
| } |
| } |
| } |
| |
| /* Determine the node_type based on type of its children. |
| * If children are all the same leaves, this internal node is a fat leaf; |
| * Otherwise, it's a mixed node. |
| */ |
| uint8_t |
| determine_internal_node_type(anv_cluster cluster, uint32_t child, uint child_count) |
| { |
| if (child_count == 0) |
| return uint8_t(ANV_NODE_TYPE_INVALID); |
| |
| uint32_t type = ir_id_to_type(child); |
| uint32_t first_type_of_child = subgroupClusteredMin(type, 8); |
| uint32_t second_type_of_child = subgroupClusteredMax(type, 8); |
| |
| if (first_type_of_child != second_type_of_child) |
| return uint8_t(ANV_NODE_TYPE_MIXED); |
| |
| /* All children have same type. Now check what type they are. */ |
| switch (first_type_of_child){ |
| case vk_ir_node_triangle: |
| return uint8_t(ANV_NODE_TYPE_QUAD); |
| case vk_ir_node_aabb: |
| return uint8_t(ANV_NODE_TYPE_PROCEDURAL); |
| case vk_ir_node_instance: |
| return uint8_t(ANV_NODE_TYPE_INSTANCE); |
| case vk_ir_node_internal: |
| return uint8_t(ANV_NODE_TYPE_MIXED); |
| default: |
| return uint8_t(ANV_NODE_TYPE_INVALID); |
| } |
| } |
| |
| vk_aabb |
| quantize_bounds(vk_aabb aabb, vec3 base, i8vec3 exp) |
| { |
| vk_aabb quant_aabb; |
| vec3 lower = aabb.min - base; |
| vec3 upper = aabb.max - base; |
| |
| vec3 qlower = ldexp(lower, -exp + 8); |
| vec3 qupper = ldexp(upper, -exp + 8); |
| |
| qlower = min(max(floor(qlower), vec3(0.0)), vec3(255.0)); |
| qupper = min(max(ceil(qupper), vec3(0.0)), vec3(255.0)); |
| |
| quant_aabb.min = qlower; |
| quant_aabb.max = qupper; |
| |
| return quant_aabb; |
| } |
| |
| void |
| encode_internal_node(uint32_t child, uint32_t child_block_offset_from_internal_node, |
| uint child_count, vk_aabb child_aabb, uint32_t bvh_block_offset, |
| anv_cluster cluster) |
| { |
| if (DEBUG_EXIT_EARLY(child_count)) |
| return; |
| REF(anv_internal_node) dst_node = |
| REF(anv_internal_node)(OFFSET(args.output_bvh, ANV_RT_BLOCK_SIZE * bvh_block_offset)); |
| |
| bool valid_leaf = cluster.idx < child_count; |
| |
| uint8_t block_incr_and_start_prim = uint8_t(0); |
| |
| if (valid_leaf) { |
| vk_aabb box; |
| box.min = subgroupClusteredMin(child_aabb.min, 8); |
| box.max = subgroupClusteredMax(child_aabb.max, 8); |
| |
| vk_aabb conservative_child_aabb = conservative_aabb(box); |
| |
| float up = 1.0 + ULP; |
| ivec3 exp; |
| |
| vec3 len = aabb_size(conservative_child_aabb) * up; |
| vec3 mant = frexp(len, exp); |
| |
| exp.x += int((mant.x > (255.0f / 256.0f))); |
| exp.y += int((mant.y > (255.0f / 256.0f))); |
| exp.z += int((mant.z > (255.0f / 256.0f))); |
| |
| i8vec3 exponent_i8 = i8vec3(exp); |
| i8vec3 exp_i8 = {max(int8_t(-128), exponent_i8.x), |
| max(int8_t(-128), exponent_i8.y), |
| max(int8_t(-128), exponent_i8.z)}; |
| |
| uint8_t node_type = determine_internal_node_type(cluster, child, child_count); |
| |
| if (cluster.idx == 0) { |
| DEREF(dst_node).child_block_offset = child_block_offset_from_internal_node; |
| DEREF(dst_node).lower[0] = conservative_child_aabb.min.x; |
| DEREF(dst_node).lower[1] = conservative_child_aabb.min.y; |
| DEREF(dst_node).lower[2] = conservative_child_aabb.min.z; |
| DEREF(dst_node).exp_x = exp_i8[0]; |
| DEREF(dst_node).exp_y = exp_i8[1]; |
| DEREF(dst_node).exp_z = exp_i8[2]; |
| DEREF(dst_node).node_mask = uint8_t(0xff); |
| DEREF(dst_node).node_type = node_type; |
| DEREF(dst_node).pad = uint8_t(0); |
| |
| if (ANV_TEST_BUILD_FLAG_WRITE_LOOKUP_MAPS_FOR_UPDATE && |
| args.parent_child_count_map != 0) { |
| DEREF(INDEX(uint8_t, args.parent_child_count_map, bvh_block_offset)) = |
| uint8_t(child_count); |
| } |
| } |
| |
| child_aabb = conservative_aabb(child_aabb); |
| vk_aabb quantize_aabb = quantize_bounds(child_aabb, conservative_child_aabb.min, exp_i8); |
| |
| DEREF(dst_node).lower_x[cluster.idx] = uint8_t(quantize_aabb.min.x); |
| DEREF(dst_node).lower_y[cluster.idx] = uint8_t(quantize_aabb.min.y); |
| DEREF(dst_node).lower_z[cluster.idx] = uint8_t(quantize_aabb.min.z); |
| DEREF(dst_node).upper_x[cluster.idx] = uint8_t(quantize_aabb.max.x); |
| DEREF(dst_node).upper_y[cluster.idx] = uint8_t(quantize_aabb.max.y); |
| DEREF(dst_node).upper_z[cluster.idx] = uint8_t(quantize_aabb.max.z); |
| |
| /* blockIncr and child_block_offset are how HW used to find children during traversal. |
| * If not set properly, gpu could hang. |
| */ |
| uint32_t type = ir_id_to_type(child); |
| block_incr_and_start_prim = type == vk_ir_node_instance ? uint8_t(2) : uint8_t(1); |
| /* for a mixed node, encode type of each children in startPrim in childdata */ |
| if (node_type == uint8_t(ANV_NODE_TYPE_MIXED)) |
| switch (type) { |
| case vk_ir_node_triangle: |
| block_incr_and_start_prim |= uint8_t(ANV_NODE_TYPE_QUAD) << 2; |
| break; |
| case vk_ir_node_aabb: |
| block_incr_and_start_prim |= uint8_t(ANV_NODE_TYPE_PROCEDURAL) << 2; |
| break; |
| case vk_ir_node_instance: |
| block_incr_and_start_prim |= uint8_t(ANV_NODE_TYPE_INSTANCE) << 2; |
| break; |
| case vk_ir_node_internal: |
| block_incr_and_start_prim |= uint8_t(ANV_NODE_TYPE_MIXED) << 2; |
| break; |
| default: |
| block_incr_and_start_prim |= uint8_t(ANV_NODE_TYPE_INVALID) << 2; |
| break; |
| } |
| } else { |
| DEREF(dst_node).lower_x[cluster.idx] = uint8_t(0x80); |
| DEREF(dst_node).lower_y[cluster.idx] = uint8_t(0); |
| DEREF(dst_node).lower_z[cluster.idx] = uint8_t(0); |
| DEREF(dst_node).upper_x[cluster.idx] = uint8_t(0); |
| DEREF(dst_node).upper_y[cluster.idx] = uint8_t(0); |
| DEREF(dst_node).upper_z[cluster.idx] = uint8_t(0); |
| } |
| DEREF(dst_node).data[cluster.idx].block_incr_and_start_prim = block_incr_and_start_prim; |
| } |
| |
| /* Collapse nodes until reaching 6 children, which typically can be |
| * 5 internal nodes, or run out of nodes to collapse which often |
| * happens at tips of tree. Tree is collapsed in direction of |
| * largest surface areas, resulting in a quality bvh tree. |
| * |
| * Early find_children phase defers node collapse as it's a |
| * speculative phase. Nodes are collapsed only if parent node |
| * is found to be real (not collapsed node.) |
| */ |
| uint32_t |
| find_children(vk_ir_box_node src, inout uint32_t children[6], |
| inout uint32_t collapsed_nodes[6], |
| out uint32_t collapsed_child_count, bool defer_collapse) |
| { |
| uint32_t found_child_count = 0; |
| collapsed_child_count = 0; |
| |
| /* Initial node can have at most two children */ |
| for (uint32_t i = 0; i < 2; ++i) |
| if (src.children[i] != VK_BVH_INVALID_NODE) |
| children[found_child_count++] = src.children[i]; |
| |
| /* For this node, try to collapse binary to 6-ary children */ |
| while (found_child_count < 6) { |
| /* find vk_ir_node_internal children with largest surface areas */ |
| int32_t collapsed_child_index = -1; |
| float largest_surface_area = -INFINITY; |
| |
| for (int32_t i = 0; i < found_child_count; ++i) { |
| /* Only collapse internal nodes, not leaf nodes. */ |
| if (ir_id_to_type(children[i]) != vk_ir_node_internal) |
| continue; |
| |
| vk_aabb bounds = DEREF(REF(vk_ir_node)NODE_OFFSET(children[i])).aabb; |
| |
| float surface_area = aabb_surface_area(bounds); |
| if (surface_area > largest_surface_area) { |
| largest_surface_area = surface_area; |
| collapsed_child_index = i; |
| } |
| } |
| |
| if (collapsed_child_index != -1) { |
| /* If deferred, save nodes to collapse later */ |
| if (defer_collapse && collapsed_child_count < 6) |
| collapsed_nodes[collapsed_child_count] = |
| ir_id_to_offset(children[collapsed_child_index]); |
| collapsed_child_count++; |
| |
| /* Once I found a good vk_ir_node_internal child, try to connect myself |
| * to this child's children, i.e. my grandchildren. Grandchildren can be |
| * internal nodes or leaves. |
| */ |
| REF(vk_ir_box_node) child_node = |
| REF(vk_ir_box_node)NODE_OFFSET(children[collapsed_child_index]); |
| IR_NODE grandchildren[2] = DEREF(child_node).children; |
| uint32_t valid_grandchild_count = 0; |
| |
| if (grandchildren[1] != VK_BVH_INVALID_NODE) |
| ++valid_grandchild_count; |
| |
| if (grandchildren[0] != VK_BVH_INVALID_NODE) |
| ++valid_grandchild_count; |
| else |
| grandchildren[0] = grandchildren[1]; |
| |
| /* Grandchild now becomes my direct child, and can possibly be collapsed |
| * in the next iteration if found_child_count has not reached 6. |
| */ |
| if (valid_grandchild_count > 1) |
| children[found_child_count++] = grandchildren[1]; |
| |
| if (valid_grandchild_count > 0) |
| children[collapsed_child_index] = grandchildren[0]; |
| else { |
| /* This child doesn't have valid children, then I don't consider this |
| * child as my child anymore. This is possible depending on how and |
| * when lbvh/ploc algorithm marks a node as VK_BVH_INVALID_NODE. |
| */ |
| found_child_count--; |
| children[collapsed_child_index] = children[found_child_count]; |
| } |
| if (!defer_collapse) |
| DEREF(child_node).bvh_offset = VK_NULL_BVH_OFFSET; |
| } else |
| break; |
| } |
| return found_child_count; |
| } |
| |
| void |
| main() |
| { |
| /* Each lane will process one vk_ir_node_internal. The root node is sitting |
| * at the end of the IR BVH, and we let the lane with |
| * gl_GlobalInvocationID.x == 0 to take care of it. |
| */ |
| uint32_t global_id = DEREF(args.header).ir_internal_node_count - |
| (gl_GlobalInvocationID.x + push_args.start_node_offset + 1); |
| |
| uint32_t intermediate_leaf_node_size = vk_ir_node_size(args.geometry_type); |
| |
| /* Each invocation cluster encodes one internal node. */ |
| anv_cluster cluster; |
| anv_cluster_init(cluster, 8); |
| |
| uint32_t intermediate_leaf_nodes_size = |
| args.leaf_node_count * intermediate_leaf_node_size; |
| |
| REF(vk_ir_box_node) intermediate_internal_nodes = |
| REF(vk_ir_box_node)OFFSET(args.intermediate_bvh, |
| intermediate_leaf_nodes_size); |
| REF(vk_ir_box_node) src_node = |
| INDEX(vk_ir_box_node, intermediate_internal_nodes, global_id); |
| vk_ir_box_node src = DEREF(src_node); |
| |
| bool is_root_node = (gl_GlobalInvocationID.x + push_args.start_node_offset) == 0; |
| |
| REF(anv_accel_struct_header) header = |
| REF(anv_accel_struct_header)(args.output_bvh - ANV_RT_BVH_HEADER_SIZE); |
| |
| if (is_root_node) { |
| /* Tracks BLOCK where the next children should be encoded. */ |
| DEREF(args.header).dst_node_offset = 1; |
| DEREF(header).instance_count = 0; |
| DEREF(header).root_flags = src.flags; |
| if (ANV_TEST_BUILD_FLAG_WRITE_LOOKUP_MAPS_FOR_UPDATE && args.parent_child_map != 0) { |
| DEREF(INDEX(uint32_t, args.parent_child_map, 0)) = VK_NULL_BVH_OFFSET; |
| } |
| } |
| |
| IR_NODE children[6] = {VK_BVH_INVALID_NODE, VK_BVH_INVALID_NODE, |
| VK_BVH_INVALID_NODE, VK_BVH_INVALID_NODE, |
| VK_BVH_INVALID_NODE, VK_BVH_INVALID_NODE}; |
| uint32_t collapsed_nodes[6]; |
| uint32_t collapsed_child_count; |
| uint32_t found_child_count; |
| uint32_t num_blocks_to_add; |
| |
| /* Every simd lane is assigned an IR BVH internal node to encode. Since |
| * we are collapsing a binary tree into a hex tree, most simd lanes will |
| * never need to encode. |
| * |
| * To increase performance, have all IR BVH speculatively calculate which |
| * nodes they would collapse. Most of this work will be thrown away since |
| * over half the IR internal nodes never get written, but reduces latency. |
| */ |
| if (ASSIGNED_NODE_TO_ENCODE) { |
| found_child_count = find_children(src, children, collapsed_nodes, |
| collapsed_child_count, true); |
| |
| /* Count the number of instance children found. For each one found, |
| * it contributes to 2 blocks to dst_node_offset |
| */ |
| num_blocks_to_add = 0; |
| for (uint32_t i = 0; i < found_child_count; ++i) { |
| uint32_t type = ir_id_to_type(children[i]); |
| num_blocks_to_add += (type == vk_ir_node_instance) ? 2 : 1; |
| } |
| } |
| |
| BLOCK bvh_block_offset = (is_root_node) ? 0 : |
| (ASSIGNED_NODE_TO_ENCODE ? VK_UNKNOWN_BVH_OFFSET |
| : VK_NULL_BVH_OFFSET); |
| |
| /* For all but the root internal node, nodes wait until their parent node |
| * informs them whether they are a valid child (valid bvh offset written) |
| * or were collapsed (VK_NULL_BVH_OFFSET written) and have no work to do. |
| */ |
| for (;;) { |
| /* Make changes to the current node's BVH offset value visible. */ |
| memoryBarrier(gl_ScopeDevice, gl_StorageSemanticsBuffer, |
| gl_SemanticsAcquire | gl_SemanticsMakeVisible); |
| |
| /* Indicate where this internal node should be encoded. Offset measured |
| * in number of 64B blocks and started from output_bvh. |
| */ |
| if (bvh_block_offset == VK_UNKNOWN_BVH_OFFSET) |
| bvh_block_offset = DEREF(src_node).bvh_offset; |
| |
| /* The invocation that processes this node is spinning, since its parent |
| * hasn't told it bvh_offset |
| */ |
| BLOCK first_child_block; |
| if (READY_TO_WRITE(bvh_block_offset)) { |
| /* Used for finding where to encode children. Also, update dst_node_offset |
| * so other invocations know where to start encoding |
| */ |
| first_child_block = |
| atomicAdd(DEREF(args.header).dst_node_offset, num_blocks_to_add); |
| |
| /* Yes, we are potentially calling find_children again here. This is to |
| * handle an edge case where some bvh trees have nodes with only a single |
| * child. This can potentially lead to lots of nodes needing to be |
| * collapsed, overflowing the 6-element buffer allocated. To handle these |
| * rare cases we find the children again, immediately collapsing them as |
| * we find them. |
| */ |
| if (collapsed_child_count > 6) |
| find_children(src, children, collapsed_nodes, collapsed_child_count, false); |
| |
| BLOCK child_offset = first_child_block; |
| for (uint32_t i = 0; i < found_child_count; ++i) { |
| /* Retrieve type and location of the child from IR BVH */ |
| uint32_t type = ir_id_to_type(children[i]); |
| |
| if (type == vk_ir_node_internal) { |
| REF(vk_ir_box_node) child_node = |
| REF(vk_ir_box_node)NODE_OFFSET(children[i]); |
| DEREF(child_node).bvh_offset = child_offset; |
| } |
| child_offset += (type == vk_ir_node_instance) ? 2 : 1; |
| } |
| |
| /* Mark this collapsed internal node as NULL, |
| * so whichever lane that would have processed it will return. |
| */ |
| for (uint32_t i = 0; i < collapsed_child_count; i++) { |
| REF(vk_ir_box_node) child_node = |
| REF(vk_ir_box_node)OFFSET(args.intermediate_bvh, collapsed_nodes[i]); |
| DEREF(child_node).bvh_offset = VK_NULL_BVH_OFFSET; |
| } |
| /* Make changes to the children's BVH offset value available to child threads. */ |
| memoryBarrier(gl_ScopeDevice, gl_StorageSemanticsBuffer, |
| gl_SemanticsRelease | gl_SemanticsMakeAvailable); |
| } |
| |
| /* While all work prior was performed with each simd lane working on |
| * separate nodes, encoding is achieved with all simd lanes in cluster |
| * working together on same internal node. Walk through each ready node |
| * and encode in concert. |
| */ |
| while (READY_TO_WRITE(subgroupClusteredMin(bvh_block_offset, 8))) { |
| /* Select next ready simd lane to write */ |
| uint32_t idx = (READY_TO_WRITE(bvh_block_offset)) ? cluster.idx : -1; |
| idx = subgroupClusteredMin(idx, 8); |
| |
| /* Propagate src child and dest blocks of next simd lane to other lanes */ |
| IR_NODE child = VK_BVH_INVALID_NODE; |
| BLOCK child_block = anv_shuffle(cluster, idx, first_child_block); |
| BLOCK internal_node_block = anv_shuffle(cluster, idx, bvh_block_offset); |
| vk_aabb child_aabb = {vec3(INFINITY), vec3(-INFINITY)}; |
| bvh_block_offset = (cluster.idx == idx) ? VK_NULL_BVH_OFFSET |
| : bvh_block_offset; |
| if (cluster.idx >= 6) |
| continue; |
| if (cluster.idx < anv_shuffle(cluster, idx, found_child_count)) { |
| for (uint32_t i = 0; ; i++) { |
| child = anv_shuffle(cluster, idx, children[i]); |
| if (i == cluster.idx) |
| break; |
| |
| /* Invalid node should not contribute to child_block count */ |
| if (child == VK_BVH_INVALID_NODE) |
| continue; |
| |
| uint32_t type = ir_id_to_type(child); |
| child_block += (type == vk_ir_node_instance) ? 2 : 1; |
| } |
| |
| if (child != VK_BVH_INVALID_NODE) |
| child_aabb = DEREF(REF(vk_ir_node)NODE_OFFSET(child)).aabb; |
| |
| uint32_t type = ir_id_to_type(child); |
| if (ANV_TEST_BUILD_FLAG_WRITE_LOOKUP_MAPS_FOR_UPDATE && |
| child != VK_BVH_INVALID_NODE && |
| args.parent_child_map != 0 && |
| args.leaf_block_offset_map != 0) { |
| if (type == vk_ir_node_triangle || type == vk_ir_node_aabb) { |
| uint32_t ir_offset = ir_id_to_offset(child); |
| uint32_t leaf_id = ir_offset / intermediate_leaf_node_size; |
| /* Block offset 0 is assigned to root, so avoid accidental |
| * assignment. |
| */ |
| DEREF(INDEX(uint32_t, args.leaf_block_offset_map, leaf_id)) = |
| (child_block != 0) ? child_block : VK_NULL_BVH_OFFSET; |
| |
| if (VK_TEST_BUILD_FLAG_HAS_QUADS && type == vk_ir_node_triangle) { |
| vk_ir_triangle_node src = DEREF(REF(vk_ir_triangle_node)NODE_OFFSET(child)); |
| vk_ir_triangle_node_quad quad = |
| vk_ir_triangle_node_get_quad(REF(vk_ir_triangle_node)NODE_OFFSET(child)); |
| if (quad.triangle_id != VK_QUAD_TRIANGLE_ID_UNUSED) { |
| uint32_t merge_id = quad.triangle_id & 0x0fffffff; |
| uint32_t merge_leaf_id = leaf_id - src.triangle_id + merge_id; |
| DEREF(INDEX(uint32_t, args.leaf_block_offset_map, merge_leaf_id)) = |
| VK_NULL_BVH_OFFSET; |
| } |
| } |
| } |
| |
| /* Track each children's parent in the map. */ |
| if (type != vk_ir_node_instance && child_block != 0) { |
| uint32_t pcm = 0; |
| pcm = internal_node_block | (cluster.idx << 26); |
| DEREF(INDEX(uint32_t, args.parent_child_map, child_block)) = pcm; |
| } |
| } |
| |
| if (child != VK_BVH_INVALID_NODE && type != vk_ir_node_internal) { |
| encode_leaf_node(type, child, BLOCK_OFFSET(child_block), |
| header); |
| } |
| } |
| |
| BLOCK child_block_offset = |
| anv_shuffle(cluster, 0, child_block) - internal_node_block; |
| encode_internal_node(child, child_block_offset, |
| anv_shuffle(cluster, idx, found_child_count), |
| child_aabb, internal_node_block, cluster); |
| } |
| |
| uint32_t is_done = (bvh_block_offset == VK_NULL_BVH_OFFSET) ? 1 : 0; |
| if (subgroupClusteredAdd(is_done, 8) == 8) |
| break; |
| } |
| |
| if (is_root_node) { |
| DEREF(header).aabb = src.base.aabb; |
| } |
| } |
