src/amd/vulkan/radv_cmd_buffer.c

/*
 * Copyright © 2016 Red Hat.
 * Copyright © 2016 Bas Nieuwenhuizen
 *
 * based in part on anv driver which is:
 * Copyright © 2015 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "radv_cs.h"
#include "radv_debug.h"
#include "radv_meta.h"
#include "radv_private.h"
#include "radv_radeon_winsys.h"
#include "radv_shader.h"
#include "sid.h"
#include "vk_format.h"
#include "vk_util.h"
#include "vk_enum_defines.h"
#include "vk_common_entrypoints.h"
#include "vk_render_pass.h"
#include "vk_framebuffer.h"

#include "ac_debug.h"
#include "ac_shader_args.h"

#include "util/fast_idiv_by_const.h"

enum {
   RADV_PREFETCH_VBO_DESCRIPTORS = (1 << 0),
   RADV_PREFETCH_VS = (1 << 1),
   RADV_PREFETCH_TCS = (1 << 2),
   RADV_PREFETCH_TES = (1 << 3),
   RADV_PREFETCH_GS = (1 << 4),
   RADV_PREFETCH_PS = (1 << 5),
   RADV_PREFETCH_MS = (1 << 6),
   RADV_PREFETCH_SHADERS = (RADV_PREFETCH_VS | RADV_PREFETCH_TCS | RADV_PREFETCH_TES |
                            RADV_PREFETCH_GS | RADV_PREFETCH_PS | RADV_PREFETCH_MS)
};

static void radv_handle_image_transition(struct radv_cmd_buffer *cmd_buffer,
                                         struct radv_image *image,
                                         VkImageLayout src_layout, VkImageLayout dst_layout,
                                         uint32_t src_family_index, uint32_t dst_family_index,
                                         const VkImageSubresourceRange *range,
                                         struct radv_sample_locations_state *sample_locs);

static void radv_set_rt_stack_size(struct radv_cmd_buffer *cmd_buffer, uint32_t size);

const struct radv_dynamic_state default_dynamic_state = {
   .viewport =
      {
         .count = 0,
      },
   .scissor =
      {
         .count = 0,
      },
   .line_width = 1.0f,
   .depth_bias =
      {
         .bias = 0.0f,
         .clamp = 0.0f,
         .slope = 0.0f,
      },
   .blend_constants = {0.0f, 0.0f, 0.0f, 0.0f},
   .depth_bounds =
      {
         .min = 0.0f,
         .max = 1.0f,
      },
   .stencil_compare_mask =
      {
         .front = ~0u,
         .back = ~0u,
      },
   .stencil_write_mask =
      {
         .front = ~0u,
         .back = ~0u,
      },
   .stencil_reference =
      {
         .front = 0u,
         .back = 0u,
      },
   .line_stipple =
      {
         .factor = 0u,
         .pattern = 0u,
      },
   .cull_mode = 0u,
   .front_face = 0u,
   .primitive_topology = 0u,
   .fragment_shading_rate =
      {
         .size = {1u, 1u},
         .combiner_ops = {VK_FRAGMENT_SHADING_RATE_COMBINER_OP_KEEP_KHR,
                          VK_FRAGMENT_SHADING_RATE_COMBINER_OP_KEEP_KHR},
      },
   .depth_bias_enable = 0u,
   .primitive_restart_enable = 0u,
   .rasterizer_discard_enable = 0u,
   .logic_op = 0u,
   .color_write_enable = 0u,
   .patch_control_points = 0,
   .polygon_mode = 0,
   .tess_domain_origin = VK_TESSELLATION_DOMAIN_ORIGIN_UPPER_LEFT,
   .logic_op_enable = 0u,
   .stippled_line_enable = 0u,
   .alpha_to_coverage_enable = 0u,
   .sample_mask = 0u,
   .depth_clip_enable = 0u,
   .conservative_rast_mode = VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT,
   .depth_clip_negative_one_to_one = 0u,
   .provoking_vertex_mode = VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT,
   .depth_clamp_enable = 0u,
};

static void
radv_bind_dynamic_state(struct radv_cmd_buffer *cmd_buffer, const struct radv_dynamic_state *src)
{
   struct radv_dynamic_state *dest = &cmd_buffer->state.dynamic;
   uint64_t copy_mask = src->mask;
   uint64_t dest_mask = 0;

   dest->discard_rectangle.count = src->discard_rectangle.count;
   dest->sample_location.count = src->sample_location.count;

   if (copy_mask & RADV_DYNAMIC_VIEWPORT) {
      if (dest->viewport.count != src->viewport.count) {
         dest->viewport.count = src->viewport.count;
         dest_mask |= RADV_DYNAMIC_VIEWPORT;
      }

      if (memcmp(&dest->viewport.viewports, &src->viewport.viewports,
                 src->viewport.count * sizeof(VkViewport))) {
         typed_memcpy(dest->viewport.viewports, src->viewport.viewports, src->viewport.count);
         typed_memcpy(dest->viewport.xform, src->viewport.xform, src->viewport.count);
         dest_mask |= RADV_DYNAMIC_VIEWPORT;
      }
   }

   if (copy_mask & RADV_DYNAMIC_SCISSOR) {
      if (dest->scissor.count != src->scissor.count) {
         dest->scissor.count = src->scissor.count;
         dest_mask |= RADV_DYNAMIC_SCISSOR;
      }

      if (memcmp(&dest->scissor.scissors, &src->scissor.scissors,
                 src->scissor.count * sizeof(VkRect2D))) {
         typed_memcpy(dest->scissor.scissors, src->scissor.scissors, src->scissor.count);
         dest_mask |= RADV_DYNAMIC_SCISSOR;
      }
   }

   if (copy_mask & RADV_DYNAMIC_BLEND_CONSTANTS) {
      if (memcmp(&dest->blend_constants, &src->blend_constants, sizeof(src->blend_constants))) {
         typed_memcpy(dest->blend_constants, src->blend_constants, 4);
         dest_mask |= RADV_DYNAMIC_BLEND_CONSTANTS;
      }
   }

   if (copy_mask & RADV_DYNAMIC_DISCARD_RECTANGLE) {
      if (memcmp(&dest->discard_rectangle.rectangles, &src->discard_rectangle.rectangles,
                 src->discard_rectangle.count * sizeof(VkRect2D))) {
         typed_memcpy(dest->discard_rectangle.rectangles, src->discard_rectangle.rectangles,
                      src->discard_rectangle.count);
         dest_mask |= RADV_DYNAMIC_DISCARD_RECTANGLE;
      }
   }

   if (copy_mask & RADV_DYNAMIC_SAMPLE_LOCATIONS) {
      if (dest->sample_location.per_pixel != src->sample_location.per_pixel ||
          dest->sample_location.grid_size.width != src->sample_location.grid_size.width ||
          dest->sample_location.grid_size.height != src->sample_location.grid_size.height ||
          memcmp(&dest->sample_location.locations, &src->sample_location.locations,
                 src->sample_location.count * sizeof(VkSampleLocationEXT))) {
         dest->sample_location.per_pixel = src->sample_location.per_pixel;
         dest->sample_location.grid_size = src->sample_location.grid_size;
         typed_memcpy(dest->sample_location.locations, src->sample_location.locations,
                      src->sample_location.count);
         dest_mask |= RADV_DYNAMIC_SAMPLE_LOCATIONS;
      }
   }

#define RADV_CMP_COPY(field, flag)                                \
   if (copy_mask & flag) {                                        \
      if (dest->field != src->field) {                            \
         dest->field = src->field;                                \
         dest_mask |= flag;                                       \
      }                                                           \
   }

   RADV_CMP_COPY(line_width, RADV_DYNAMIC_LINE_WIDTH);

   RADV_CMP_COPY(depth_bias.bias, RADV_DYNAMIC_DEPTH_BIAS);
   RADV_CMP_COPY(depth_bias.clamp, RADV_DYNAMIC_DEPTH_BIAS);
   RADV_CMP_COPY(depth_bias.slope, RADV_DYNAMIC_DEPTH_BIAS);

   RADV_CMP_COPY(depth_bounds.min, RADV_DYNAMIC_DEPTH_BOUNDS);
   RADV_CMP_COPY(depth_bounds.max, RADV_DYNAMIC_DEPTH_BOUNDS);

   RADV_CMP_COPY(stencil_compare_mask.front, RADV_DYNAMIC_STENCIL_COMPARE_MASK);
   RADV_CMP_COPY(stencil_compare_mask.back, RADV_DYNAMIC_STENCIL_COMPARE_MASK);

   RADV_CMP_COPY(stencil_write_mask.front, RADV_DYNAMIC_STENCIL_WRITE_MASK);
   RADV_CMP_COPY(stencil_write_mask.back, RADV_DYNAMIC_STENCIL_WRITE_MASK);

   RADV_CMP_COPY(stencil_reference.front, RADV_DYNAMIC_STENCIL_REFERENCE);
   RADV_CMP_COPY(stencil_reference.back, RADV_DYNAMIC_STENCIL_REFERENCE);

   RADV_CMP_COPY(line_stipple.factor, RADV_DYNAMIC_LINE_STIPPLE);
   RADV_CMP_COPY(line_stipple.pattern, RADV_DYNAMIC_LINE_STIPPLE);

   RADV_CMP_COPY(cull_mode, RADV_DYNAMIC_CULL_MODE);
   RADV_CMP_COPY(front_face, RADV_DYNAMIC_FRONT_FACE);
   RADV_CMP_COPY(primitive_topology, RADV_DYNAMIC_PRIMITIVE_TOPOLOGY);
   RADV_CMP_COPY(depth_test_enable, RADV_DYNAMIC_DEPTH_TEST_ENABLE);
   RADV_CMP_COPY(depth_write_enable, RADV_DYNAMIC_DEPTH_WRITE_ENABLE);
   RADV_CMP_COPY(depth_compare_op, RADV_DYNAMIC_DEPTH_COMPARE_OP);
   RADV_CMP_COPY(depth_bounds_test_enable, RADV_DYNAMIC_DEPTH_BOUNDS_TEST_ENABLE);
   RADV_CMP_COPY(stencil_test_enable, RADV_DYNAMIC_STENCIL_TEST_ENABLE);

   RADV_CMP_COPY(stencil_op.front.fail_op, RADV_DYNAMIC_STENCIL_OP);
   RADV_CMP_COPY(stencil_op.front.pass_op, RADV_DYNAMIC_STENCIL_OP);
   RADV_CMP_COPY(stencil_op.front.depth_fail_op, RADV_DYNAMIC_STENCIL_OP);
   RADV_CMP_COPY(stencil_op.front.compare_op, RADV_DYNAMIC_STENCIL_OP);
   RADV_CMP_COPY(stencil_op.back.fail_op, RADV_DYNAMIC_STENCIL_OP);
   RADV_CMP_COPY(stencil_op.back.pass_op, RADV_DYNAMIC_STENCIL_OP);
   RADV_CMP_COPY(stencil_op.back.depth_fail_op, RADV_DYNAMIC_STENCIL_OP);
   RADV_CMP_COPY(stencil_op.back.compare_op, RADV_DYNAMIC_STENCIL_OP);

   RADV_CMP_COPY(fragment_shading_rate.size.width, RADV_DYNAMIC_FRAGMENT_SHADING_RATE);
   RADV_CMP_COPY(fragment_shading_rate.size.height, RADV_DYNAMIC_FRAGMENT_SHADING_RATE);
   RADV_CMP_COPY(fragment_shading_rate.combiner_ops[0], RADV_DYNAMIC_FRAGMENT_SHADING_RATE);
   RADV_CMP_COPY(fragment_shading_rate.combiner_ops[1], RADV_DYNAMIC_FRAGMENT_SHADING_RATE);

   RADV_CMP_COPY(depth_bias_enable, RADV_DYNAMIC_DEPTH_BIAS_ENABLE);

   RADV_CMP_COPY(primitive_restart_enable, RADV_DYNAMIC_PRIMITIVE_RESTART_ENABLE);

   RADV_CMP_COPY(rasterizer_discard_enable, RADV_DYNAMIC_RASTERIZER_DISCARD_ENABLE);

   RADV_CMP_COPY(logic_op, RADV_DYNAMIC_LOGIC_OP);

   RADV_CMP_COPY(color_write_enable, RADV_DYNAMIC_COLOR_WRITE_ENABLE);

   RADV_CMP_COPY(patch_control_points, RADV_DYNAMIC_PATCH_CONTROL_POINTS);

   RADV_CMP_COPY(polygon_mode, RADV_DYNAMIC_POLYGON_MODE);

   RADV_CMP_COPY(tess_domain_origin, RADV_DYNAMIC_TESS_DOMAIN_ORIGIN);

   RADV_CMP_COPY(logic_op_enable, RADV_DYNAMIC_LOGIC_OP_ENABLE);

   RADV_CMP_COPY(stippled_line_enable, RADV_DYNAMIC_LINE_STIPPLE_ENABLE);

   RADV_CMP_COPY(alpha_to_coverage_enable, RADV_DYNAMIC_ALPHA_TO_COVERAGE_ENABLE);

   RADV_CMP_COPY(sample_mask, RADV_DYNAMIC_SAMPLE_MASK);

   RADV_CMP_COPY(depth_clip_enable, RADV_DYNAMIC_DEPTH_CLIP_ENABLE);

   RADV_CMP_COPY(conservative_rast_mode, RADV_DYNAMIC_CONSERVATIVE_RAST_MODE);

   RADV_CMP_COPY(depth_clip_negative_one_to_one, RADV_DYNAMIC_DEPTH_CLIP_NEGATIVE_ONE_TO_ONE);

   RADV_CMP_COPY(provoking_vertex_mode, RADV_DYNAMIC_PROVOKING_VERTEX_MODE);

   RADV_CMP_COPY(depth_clamp_enable, RADV_DYNAMIC_DEPTH_CLAMP_ENABLE);

#undef RADV_CMP_COPY

   cmd_buffer->state.dirty |= dest_mask;
}

bool
radv_cmd_buffer_uses_mec(struct radv_cmd_buffer *cmd_buffer)
{
   return cmd_buffer->qf == RADV_QUEUE_COMPUTE &&
          cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX7;
}

enum amd_ip_type
radv_queue_family_to_ring(struct radv_physical_device *physical_device,
                          enum radv_queue_family f)
{
   switch (f) {
   case RADV_QUEUE_GENERAL:
      return AMD_IP_GFX;
   case RADV_QUEUE_COMPUTE:
      return AMD_IP_COMPUTE;
   case RADV_QUEUE_TRANSFER:
      return AMD_IP_SDMA;
   default:
      unreachable("Unknown queue family");
   }
}

static void
radv_emit_write_data_packet(struct radv_cmd_buffer *cmd_buffer, unsigned engine_sel, uint64_t va,
                            unsigned count, const uint32_t *data)
{
   struct radeon_cmdbuf *cs = cmd_buffer->cs;

   radeon_check_space(cmd_buffer->device->ws, cs, 4 + count);

   radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + count, 0));
   radeon_emit(cs, S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(engine_sel));
   radeon_emit(cs, va);
   radeon_emit(cs, va >> 32);
   radeon_emit_array(cs, data, count);
}

static void
radv_emit_clear_data(struct radv_cmd_buffer *cmd_buffer, unsigned engine_sel, uint64_t va,
                     unsigned size)
{
   uint32_t *zeroes = alloca(size);
   memset(zeroes, 0, size);
   radv_emit_write_data_packet(cmd_buffer, engine_sel, va, size / 4, zeroes);
}

static void
radv_destroy_cmd_buffer(struct vk_command_buffer *vk_cmd_buffer)
{
   struct radv_cmd_buffer *cmd_buffer = container_of(vk_cmd_buffer, struct radv_cmd_buffer, vk);

   list_for_each_entry_safe(struct radv_cmd_buffer_upload, up, &cmd_buffer->upload.list, list)
   {
      cmd_buffer->device->ws->buffer_destroy(cmd_buffer->device->ws, up->upload_bo);
      list_del(&up->list);
      free(up);
   }

   if (cmd_buffer->upload.upload_bo)
      cmd_buffer->device->ws->buffer_destroy(cmd_buffer->device->ws, cmd_buffer->upload.upload_bo);

   if (cmd_buffer->cs)
      cmd_buffer->device->ws->cs_destroy(cmd_buffer->cs);
   if (cmd_buffer->ace_internal.cs)
      cmd_buffer->device->ws->cs_destroy(cmd_buffer->ace_internal.cs);

   for (unsigned i = 0; i < MAX_BIND_POINTS; i++) {
      struct radv_descriptor_set_header *set = &cmd_buffer->descriptors[i].push_set.set;
      free(set->mapped_ptr);
      if (set->layout)
         vk_descriptor_set_layout_unref(&cmd_buffer->device->vk, &set->layout->vk);
      vk_object_base_finish(&set->base);
   }

   vk_object_base_finish(&cmd_buffer->meta_push_descriptors.base);

   vk_command_buffer_finish(&cmd_buffer->vk);
   vk_free(&cmd_buffer->vk.pool->alloc, cmd_buffer);
}

static VkResult
radv_create_cmd_buffer(struct vk_command_pool *pool,
                       struct vk_command_buffer **cmd_buffer_out)
{
   struct radv_device *device = container_of(pool->base.device, struct radv_device, vk);

   struct radv_cmd_buffer *cmd_buffer;
   unsigned ring;
   cmd_buffer = vk_zalloc(&pool->alloc, sizeof(*cmd_buffer), 8,
                          VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
   if (cmd_buffer == NULL)
      return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);

   VkResult result =
      vk_command_buffer_init(pool, &cmd_buffer->vk, &radv_cmd_buffer_ops, 0);
   if (result != VK_SUCCESS) {
      vk_free(&cmd_buffer->vk.pool->alloc, cmd_buffer);
      return result;
   }

   cmd_buffer->device = device;

   cmd_buffer->qf = vk_queue_to_radv(device->physical_device, pool->queue_family_index);

   ring = radv_queue_family_to_ring(device->physical_device, cmd_buffer->qf);

   cmd_buffer->cs = device->ws->cs_create(device->ws, ring);
   if (!cmd_buffer->cs) {
      radv_destroy_cmd_buffer(&cmd_buffer->vk);
      return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
   }

   vk_object_base_init(&device->vk, &cmd_buffer->meta_push_descriptors.base,
                       VK_OBJECT_TYPE_DESCRIPTOR_SET);

   for (unsigned i = 0; i < MAX_BIND_POINTS; i++)
      vk_object_base_init(&device->vk, &cmd_buffer->descriptors[i].push_set.set.base,
                          VK_OBJECT_TYPE_DESCRIPTOR_SET);

   *cmd_buffer_out = &cmd_buffer->vk;

   list_inithead(&cmd_buffer->upload.list);

   return VK_SUCCESS;
}

void
radv_cmd_buffer_reset_rendering(struct radv_cmd_buffer *cmd_buffer)
{
   memset(&cmd_buffer->state.render, 0, sizeof(cmd_buffer->state.render));
}

static void
radv_reset_cmd_buffer(struct vk_command_buffer *vk_cmd_buffer,
                      UNUSED VkCommandBufferResetFlags flags)
{
   struct radv_cmd_buffer *cmd_buffer = container_of(vk_cmd_buffer, struct radv_cmd_buffer, vk);

   vk_command_buffer_reset(&cmd_buffer->vk);

   cmd_buffer->device->ws->cs_reset(cmd_buffer->cs);
   if (cmd_buffer->ace_internal.cs)
      cmd_buffer->device->ws->cs_reset(cmd_buffer->ace_internal.cs);

   list_for_each_entry_safe(struct radv_cmd_buffer_upload, up, &cmd_buffer->upload.list, list)
   {
      cmd_buffer->device->ws->buffer_destroy(cmd_buffer->device->ws, up->upload_bo);
      list_del(&up->list);
      free(up);
   }

   cmd_buffer->push_constant_stages = 0;
   cmd_buffer->scratch_size_per_wave_needed = 0;
   cmd_buffer->scratch_waves_wanted = 0;
   cmd_buffer->compute_scratch_size_per_wave_needed = 0;
   cmd_buffer->compute_scratch_waves_wanted = 0;
   cmd_buffer->esgs_ring_size_needed = 0;
   cmd_buffer->gsvs_ring_size_needed = 0;
   cmd_buffer->tess_rings_needed = false;
   cmd_buffer->task_rings_needed = false;
   cmd_buffer->mesh_scratch_ring_needed = false;
   cmd_buffer->gds_needed = false;
   cmd_buffer->gds_oa_needed = false;
   cmd_buffer->sample_positions_needed = false;
   cmd_buffer->ace_internal.sem.gfx2ace_value = 0;
   cmd_buffer->ace_internal.sem.emitted_gfx2ace_value = 0;
   cmd_buffer->ace_internal.sem.va = 0;

   if (cmd_buffer->upload.upload_bo)
      radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, cmd_buffer->upload.upload_bo);
   cmd_buffer->upload.offset = 0;

   memset(cmd_buffer->vertex_binding_buffers, 0, sizeof(struct radv_buffer *) * cmd_buffer->used_vertex_bindings);
   cmd_buffer->used_vertex_bindings = 0;

   for (unsigned i = 0; i < MAX_BIND_POINTS; i++) {
      cmd_buffer->descriptors[i].dirty = 0;
      cmd_buffer->descriptors[i].valid = 0;
      cmd_buffer->descriptors[i].push_dirty = false;
   }

   if (cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX7) {
      uint32_t pred_value = 0;
      uint32_t pred_offset;
      if (!radv_cmd_buffer_upload_data(cmd_buffer, 4, &pred_value, &pred_offset))
         vk_command_buffer_set_error(&cmd_buffer->vk, VK_ERROR_OUT_OF_HOST_MEMORY);

      cmd_buffer->mec_inv_pred_emitted = false;
      cmd_buffer->mec_inv_pred_va = radv_buffer_get_va(cmd_buffer->upload.upload_bo) + pred_offset;
   }

   if (cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX9 &&
       cmd_buffer->qf == RADV_QUEUE_GENERAL) {
      unsigned num_db = cmd_buffer->device->physical_device->rad_info.max_render_backends;
      unsigned fence_offset, eop_bug_offset;
      void *fence_ptr;

      radv_cmd_buffer_upload_alloc(cmd_buffer, 8, &fence_offset, &fence_ptr);
      memset(fence_ptr, 0, 8);

      cmd_buffer->gfx9_fence_va = radv_buffer_get_va(cmd_buffer->upload.upload_bo);
      cmd_buffer->gfx9_fence_va += fence_offset;

      radv_emit_clear_data(cmd_buffer, V_370_PFP, cmd_buffer->gfx9_fence_va, 8);

      if (cmd_buffer->device->physical_device->rad_info.gfx_level == GFX9) {
         /* Allocate a buffer for the EOP bug on GFX9. */
         radv_cmd_buffer_upload_alloc(cmd_buffer, 16 * num_db, &eop_bug_offset, &fence_ptr);
         memset(fence_ptr, 0, 16 * num_db);
         cmd_buffer->gfx9_eop_bug_va = radv_buffer_get_va(cmd_buffer->upload.upload_bo);
         cmd_buffer->gfx9_eop_bug_va += eop_bug_offset;

         radv_emit_clear_data(cmd_buffer, V_370_PFP, cmd_buffer->gfx9_eop_bug_va, 16 * num_db);
      }
   }

   radv_cmd_buffer_reset_rendering(cmd_buffer);

   cmd_buffer->status = RADV_CMD_BUFFER_STATUS_INITIAL;
}

const struct vk_command_buffer_ops radv_cmd_buffer_ops = {
   .create = radv_create_cmd_buffer,
   .reset = radv_reset_cmd_buffer,
   .destroy = radv_destroy_cmd_buffer,
};

static bool
radv_cmd_buffer_resize_upload_buf(struct radv_cmd_buffer *cmd_buffer, uint64_t min_needed)
{
   uint64_t new_size;
   struct radeon_winsys_bo *bo = NULL;
   struct radv_cmd_buffer_upload *upload;
   struct radv_device *device = cmd_buffer->device;

   new_size = MAX2(min_needed, 16 * 1024);
   new_size = MAX2(new_size, 2 * cmd_buffer->upload.size);

   VkResult result =
      device->ws->buffer_create(device->ws, new_size, 4096, device->ws->cs_domain(device->ws),
                                RADEON_FLAG_CPU_ACCESS | RADEON_FLAG_NO_INTERPROCESS_SHARING |
                                   RADEON_FLAG_32BIT | RADEON_FLAG_GTT_WC,
                                RADV_BO_PRIORITY_UPLOAD_BUFFER, 0, &bo);

   if (result != VK_SUCCESS) {
      vk_command_buffer_set_error(&cmd_buffer->vk, result);
      return false;
   }

   radv_cs_add_buffer(device->ws, cmd_buffer->cs, bo);
   if (cmd_buffer->upload.upload_bo) {
      upload = malloc(sizeof(*upload));

      if (!upload) {
         vk_command_buffer_set_error(&cmd_buffer->vk, VK_ERROR_OUT_OF_HOST_MEMORY);
         device->ws->buffer_destroy(device->ws, bo);
         return false;
      }

      memcpy(upload, &cmd_buffer->upload, sizeof(*upload));
      list_add(&upload->list, &cmd_buffer->upload.list);
   }

   cmd_buffer->upload.upload_bo = bo;
   cmd_buffer->upload.size = new_size;
   cmd_buffer->upload.offset = 0;
   cmd_buffer->upload.map = device->ws->buffer_map(cmd_buffer->upload.upload_bo);

   if (!cmd_buffer->upload.map) {
      vk_command_buffer_set_error(&cmd_buffer->vk, VK_ERROR_OUT_OF_DEVICE_MEMORY);
      return false;
   }

   return true;
}

bool
radv_cmd_buffer_upload_alloc(struct radv_cmd_buffer *cmd_buffer, unsigned size,
                             unsigned *out_offset, void **ptr)
{
   assert(size % 4 == 0);

   struct radeon_info *rad_info = &cmd_buffer->device->physical_device->rad_info;

   /* Align to the scalar cache line size if it results in this allocation
    * being placed in less of them.
    */
   unsigned offset = cmd_buffer->upload.offset;
   unsigned line_size = rad_info->gfx_level >= GFX10 ? 64 : 32;
   unsigned gap = align(offset, line_size) - offset;
   if ((size & (line_size - 1)) > gap)
      offset = align(offset, line_size);

   if (offset + size > cmd_buffer->upload.size) {
      if (!radv_cmd_buffer_resize_upload_buf(cmd_buffer, size))
         return false;
      offset = 0;
   }

   *out_offset = offset;
   *ptr = cmd_buffer->upload.map + offset;

   cmd_buffer->upload.offset = offset + size;
   return true;
}

bool
radv_cmd_buffer_upload_data(struct radv_cmd_buffer *cmd_buffer, unsigned size, const void *data,
                            unsigned *out_offset)
{
   uint8_t *ptr;

   if (!radv_cmd_buffer_upload_alloc(cmd_buffer, size, out_offset, (void **)&ptr))
      return false;
   assert(ptr);

   memcpy(ptr, data, size);
   return true;
}

void
radv_cmd_buffer_trace_emit(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_device *device = cmd_buffer->device;
   struct radeon_cmdbuf *cs = cmd_buffer->cs;
   uint64_t va;

   va = radv_buffer_get_va(device->trace_bo);
   if (cmd_buffer->vk.level == VK_COMMAND_BUFFER_LEVEL_SECONDARY)
      va += 4;

   ++cmd_buffer->state.trace_id;
   radv_emit_write_data_packet(cmd_buffer, V_370_ME, va, 1, &cmd_buffer->state.trace_id);

   radeon_check_space(cmd_buffer->device->ws, cs, 2);

   radeon_emit(cs, PKT3(PKT3_NOP, 0, 0));
   radeon_emit(cs, AC_ENCODE_TRACE_POINT(cmd_buffer->state.trace_id));
}

static void
radv_ace_internal_barrier(struct radv_cmd_buffer *cmd_buffer, VkPipelineStageFlags2 src_stage_mask,
                          VkPipelineStageFlags2 dst_stage_mask)
{
   /* Update flush bits from the main cmdbuf, except the stage flush. */
   cmd_buffer->ace_internal.flush_bits |=
      cmd_buffer->state.flush_bits & RADV_CMD_FLUSH_ALL_COMPUTE & ~RADV_CMD_FLAG_CS_PARTIAL_FLUSH;

   /* Add stage flush only when necessary. */
   if (src_stage_mask &
       (VK_PIPELINE_STAGE_2_TASK_SHADER_BIT_EXT | VK_PIPELINE_STAGE_2_TRANSFER_BIT |
        VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT))
      cmd_buffer->ace_internal.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH;

   /* Block task shaders when we have to wait for CP DMA on the GFX cmdbuf. */
   if (src_stage_mask &
       (VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_CLEAR_BIT |
        VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT | VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT |
        VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT))
      dst_stage_mask |= cmd_buffer->state.dma_is_busy ? VK_PIPELINE_STAGE_2_TASK_SHADER_BIT_EXT : 0;

   /* Increment the GFX/ACE semaphore when task shaders are blocked. */
   if (dst_stage_mask &
       (VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT_KHR | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT |
        VK_PIPELINE_STAGE_2_TASK_SHADER_BIT_EXT))
      cmd_buffer->ace_internal.sem.gfx2ace_value++;
}

static void
radv_ace_internal_cache_flush(struct radv_cmd_buffer *cmd_buffer)
{
   struct radeon_cmdbuf *ace_cs = cmd_buffer->ace_internal.cs;
   const uint32_t flush_bits = cmd_buffer->ace_internal.flush_bits;
   enum rgp_flush_bits sqtt_flush_bits = 0;

   si_cs_emit_cache_flush(ace_cs, cmd_buffer->device->physical_device->rad_info.gfx_level, NULL, 0,
                          true, flush_bits, &sqtt_flush_bits, 0);

   cmd_buffer->ace_internal.flush_bits = 0;
}

static uint64_t
radv_ace_internal_sem_create(struct radv_cmd_buffer *cmd_buffer)
{
   /* DWORD 0: GFX->ACE semaphore (GFX blocks ACE, ie. ACE waits for GFX)
    * DWORD 1: ACE->GFX semaphore
    */
   uint64_t sem_init = 0;
   uint32_t va_off = 0;
   if (!radv_cmd_buffer_upload_data(cmd_buffer, sizeof(uint64_t), &sem_init, &va_off)) {
      vk_command_buffer_set_error(&cmd_buffer->vk, VK_ERROR_OUT_OF_HOST_MEMORY);
      return 0;
   }

   return radv_buffer_get_va(cmd_buffer->upload.upload_bo) + va_off;
}

static bool
radv_ace_internal_sem_dirty(const struct radv_cmd_buffer *cmd_buffer)
{
   return cmd_buffer->ace_internal.sem.gfx2ace_value !=
          cmd_buffer->ace_internal.sem.emitted_gfx2ace_value;
}

ALWAYS_INLINE static bool
radv_flush_gfx2ace_semaphore(struct radv_cmd_buffer *cmd_buffer)
{
   if (!radv_ace_internal_sem_dirty(cmd_buffer))
      return false;

   if (!cmd_buffer->ace_internal.sem.va) {
      cmd_buffer->ace_internal.sem.va = radv_ace_internal_sem_create(cmd_buffer);
      if (!cmd_buffer->ace_internal.sem.va)
         return false;
   }

   /* GFX writes a value to the semaphore which ACE can wait for.*/
   si_cs_emit_write_event_eop(
      cmd_buffer->cs, cmd_buffer->device->physical_device->rad_info.gfx_level,
      radv_cmd_buffer_uses_mec(cmd_buffer), V_028A90_BOTTOM_OF_PIPE_TS, 0, EOP_DST_SEL_MEM,
      EOP_DATA_SEL_VALUE_32BIT, cmd_buffer->ace_internal.sem.va,
      cmd_buffer->ace_internal.sem.gfx2ace_value, cmd_buffer->gfx9_eop_bug_va);

   cmd_buffer->ace_internal.sem.emitted_gfx2ace_value = cmd_buffer->ace_internal.sem.gfx2ace_value;
   return true;
}

ALWAYS_INLINE static void
radv_wait_gfx2ace_semaphore(struct radv_cmd_buffer *cmd_buffer)
{
   assert(cmd_buffer->ace_internal.sem.va);
   struct radeon_cmdbuf *ace_cs = cmd_buffer->ace_internal.cs;
   radeon_check_space(cmd_buffer->device->ws, ace_cs, 7);

   /* ACE waits for the semaphore which GFX wrote. */
   radv_cp_wait_mem(ace_cs, WAIT_REG_MEM_GREATER_OR_EQUAL, cmd_buffer->ace_internal.sem.va,
                    cmd_buffer->ace_internal.sem.gfx2ace_value, 0xffffffff);
}

static struct radeon_cmdbuf *
radv_ace_internal_create(struct radv_cmd_buffer *cmd_buffer)
{
   assert(!cmd_buffer->ace_internal.cs);
   struct radv_device *device = cmd_buffer->device;
   struct radeon_cmdbuf *ace_cs = device->ws->cs_create(device->ws, AMD_IP_COMPUTE);

   if (!ace_cs)
      vk_command_buffer_set_error(&cmd_buffer->vk, VK_ERROR_OUT_OF_HOST_MEMORY);

   return ace_cs;
}

static VkResult
radv_ace_internal_finalize(struct radv_cmd_buffer *cmd_buffer)
{
   assert(cmd_buffer->ace_internal.cs);
   struct radv_device *device = cmd_buffer->device;
   struct radeon_cmdbuf *ace_cs = cmd_buffer->ace_internal.cs;

   /* Emit pending cache flush. */
   radv_ace_internal_cache_flush(cmd_buffer);

   /* Clear the ACE semaphore if it exists.
    * This is necessary in case the same cmd buffer is submitted again in the future.
    */
   if (cmd_buffer->ace_internal.sem.va) {
      struct radeon_cmdbuf *main_cs = cmd_buffer->cs;
      uint64_t gfx2ace_va = cmd_buffer->ace_internal.sem.va;
      uint64_t ace2gfx_va = cmd_buffer->ace_internal.sem.va + 4;

      /* ACE: write 1 to the ACE->GFX semaphore. */
      si_cs_emit_write_event_eop(ace_cs, cmd_buffer->device->physical_device->rad_info.gfx_level,
                                 true, V_028A90_BOTTOM_OF_PIPE_TS, 0, EOP_DST_SEL_MEM,
                                 EOP_DATA_SEL_VALUE_32BIT, ace2gfx_va, 1,
                                 cmd_buffer->gfx9_eop_bug_va);

      /* Wait for ACE to finish, otherwise we may risk writing 0 to the semaphore
       * when ACE is still waiting for it. This may not happen in practice, but
       * better safe than sorry.
       */
      radv_cp_wait_mem(main_cs, WAIT_REG_MEM_GREATER_OR_EQUAL, ace2gfx_va, 1, 0xffffffff);

      /* GFX: clear GFX->ACE and ACE->GFX semaphores. */
      radv_emit_clear_data(cmd_buffer, V_370_ME, gfx2ace_va, 8);
   }

   device->ws->cs_add_buffers(ace_cs, cmd_buffer->cs);
   return device->ws->cs_finalize(ace_cs);
}

static void
radv_cmd_buffer_after_draw(struct radv_cmd_buffer *cmd_buffer, enum radv_cmd_flush_bits flags)
{
   if (unlikely(cmd_buffer->device->thread_trace.bo)) {
      radeon_emit(cmd_buffer->cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
      radeon_emit(cmd_buffer->cs, EVENT_TYPE(V_028A90_THREAD_TRACE_MARKER) | EVENT_INDEX(0));
   }

   if (cmd_buffer->device->instance->debug_flags & RADV_DEBUG_SYNC_SHADERS) {
      enum rgp_flush_bits sqtt_flush_bits = 0;
      assert(flags & (RADV_CMD_FLAG_PS_PARTIAL_FLUSH | RADV_CMD_FLAG_CS_PARTIAL_FLUSH));

      radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 4);

      /* Force wait for graphics or compute engines to be idle. */
      si_cs_emit_cache_flush(cmd_buffer->cs,
                             cmd_buffer->device->physical_device->rad_info.gfx_level,
                             &cmd_buffer->gfx9_fence_idx, cmd_buffer->gfx9_fence_va,
                             radv_cmd_buffer_uses_mec(cmd_buffer), flags, &sqtt_flush_bits,
                             cmd_buffer->gfx9_eop_bug_va);

      if (cmd_buffer->state.graphics_pipeline && (flags & RADV_CMD_FLAG_PS_PARTIAL_FLUSH) &&
          radv_pipeline_has_stage(cmd_buffer->state.graphics_pipeline, MESA_SHADER_TASK)) {
         /* Force wait for compute engines to be idle on the internal cmdbuf. */
         si_cs_emit_cache_flush(cmd_buffer->ace_internal.cs,
                                cmd_buffer->device->physical_device->rad_info.gfx_level, NULL, 0,
                                true, RADV_CMD_FLAG_CS_PARTIAL_FLUSH, &sqtt_flush_bits, 0);
      }
   }

   if (unlikely(cmd_buffer->device->trace_bo))
      radv_cmd_buffer_trace_emit(cmd_buffer);
}

static void
radv_save_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline)
{
   struct radv_device *device = cmd_buffer->device;
   enum amd_ip_type ring;
   uint32_t data[2];
   uint64_t va;

   va = radv_buffer_get_va(device->trace_bo);

   ring = radv_queue_family_to_ring(device->physical_device, cmd_buffer->qf);

   switch (ring) {
   case AMD_IP_GFX:
      va += 8;
      break;
   case AMD_IP_COMPUTE:
      va += 16;
      break;
   default:
      assert(!"invalid IP type");
   }

   uint64_t pipeline_address = (uintptr_t)pipeline;
   data[0] = pipeline_address;
   data[1] = pipeline_address >> 32;

   radv_emit_write_data_packet(cmd_buffer, V_370_ME, va, 2, data);
}

static void
radv_save_vertex_descriptors(struct radv_cmd_buffer *cmd_buffer, uint64_t vb_ptr)
{
   struct radv_device *device = cmd_buffer->device;
   uint32_t data[2];
   uint64_t va;

   va = radv_buffer_get_va(device->trace_bo);
   va += 24;

   data[0] = vb_ptr;
   data[1] = vb_ptr >> 32;

   radv_emit_write_data_packet(cmd_buffer, V_370_ME, va, 2, data);
}

static void
radv_save_vs_prolog(struct radv_cmd_buffer *cmd_buffer, const struct radv_shader_part *prolog)
{
   struct radv_device *device = cmd_buffer->device;
   uint32_t data[2];
   uint64_t va;

   va = radv_buffer_get_va(device->trace_bo);
   va += 32;

   uint64_t prolog_address = (uintptr_t)prolog;
   data[0] = prolog_address;
   data[1] = prolog_address >> 32;

   radv_emit_write_data_packet(cmd_buffer, V_370_ME, va, 2, data);
}

void
radv_set_descriptor_set(struct radv_cmd_buffer *cmd_buffer, VkPipelineBindPoint bind_point,
                        struct radv_descriptor_set *set, unsigned idx)
{
   struct radv_descriptor_state *descriptors_state =
      radv_get_descriptors_state(cmd_buffer, bind_point);

   descriptors_state->sets[idx] = set;

   descriptors_state->valid |= (1u << idx); /* active descriptors */
   descriptors_state->dirty |= (1u << idx);
}

static void
radv_save_descriptors(struct radv_cmd_buffer *cmd_buffer, VkPipelineBindPoint bind_point)
{
   struct radv_descriptor_state *descriptors_state =
      radv_get_descriptors_state(cmd_buffer, bind_point);
   struct radv_device *device = cmd_buffer->device;
   uint32_t data[MAX_SETS * 2] = {0};
   uint64_t va;
   va = radv_buffer_get_va(device->trace_bo) + 40;

   u_foreach_bit(i, descriptors_state->valid)
   {
      struct radv_descriptor_set *set = descriptors_state->sets[i];
      data[i * 2] = (uint64_t)(uintptr_t)set;
      data[i * 2 + 1] = (uint64_t)(uintptr_t)set >> 32;
   }

   radv_emit_write_data_packet(cmd_buffer, V_370_ME, va, MAX_SETS * 2, data);
}

struct radv_userdata_info *
radv_lookup_user_sgpr(const struct radv_pipeline *pipeline, gl_shader_stage stage, int idx)
{
   struct radv_shader *shader = radv_get_shader(pipeline, stage);
   return &shader->info.user_sgprs_locs.shader_data[idx];
}

static void
radv_emit_userdata_address(struct radv_device *device, struct radeon_cmdbuf *cs,
                           struct radv_pipeline *pipeline, gl_shader_stage stage, int idx,
                           uint64_t va)
{
   struct radv_userdata_info *loc = radv_lookup_user_sgpr(pipeline, stage, idx);
   uint32_t base_reg = pipeline->user_data_0[stage];
   if (loc->sgpr_idx == -1)
      return;

   assert(loc->num_sgprs == 1);

   radv_emit_shader_pointer(device, cs, base_reg + loc->sgpr_idx * 4, va, false);
}

static void
radv_emit_descriptor_pointers(struct radv_device *device, struct radeon_cmdbuf *cs,
                              struct radv_pipeline *pipeline,
                              struct radv_descriptor_state *descriptors_state,
                              gl_shader_stage stage)
{
   uint32_t sh_base = pipeline->user_data_0[stage];
   struct radv_userdata_locations *locs = &pipeline->shaders[stage]->info.user_sgprs_locs;
   unsigned mask = locs->descriptor_sets_enabled;

   mask &= descriptors_state->dirty & descriptors_state->valid;

   while (mask) {
      int start, count;

      u_bit_scan_consecutive_range(&mask, &start, &count);

      struct radv_userdata_info *loc = &locs->descriptor_sets[start];
      unsigned sh_offset = sh_base + loc->sgpr_idx * 4;

      radv_emit_shader_pointer_head(cs, sh_offset, count, true);
      for (int i = 0; i < count; i++) {
         struct radv_descriptor_set *set = descriptors_state->sets[start + i];

         radv_emit_shader_pointer_body(device, cs, set->header.va, true);
      }
   }
}

/**
 * Convert the user sample locations to hardware sample locations (the values
 * that will be emitted by PA_SC_AA_SAMPLE_LOCS_PIXEL_*).
 */
static void
radv_convert_user_sample_locs(struct radv_sample_locations_state *state, uint32_t x, uint32_t y,
                              VkOffset2D *sample_locs)
{
   uint32_t x_offset = x % state->grid_size.width;
   uint32_t y_offset = y % state->grid_size.height;
   uint32_t num_samples = (uint32_t)state->per_pixel;
   VkSampleLocationEXT *user_locs;
   uint32_t pixel_offset;

   pixel_offset = (x_offset + y_offset * state->grid_size.width) * num_samples;

   assert(pixel_offset <= MAX_SAMPLE_LOCATIONS);
   user_locs = &state->locations[pixel_offset];

   for (uint32_t i = 0; i < num_samples; i++) {
      float shifted_pos_x = user_locs[i].x - 0.5;
      float shifted_pos_y = user_locs[i].y - 0.5;

      int32_t scaled_pos_x = floorf(shifted_pos_x * 16);
      int32_t scaled_pos_y = floorf(shifted_pos_y * 16);

      sample_locs[i].x = CLAMP(scaled_pos_x, -8, 7);
      sample_locs[i].y = CLAMP(scaled_pos_y, -8, 7);
   }
}

/**
 * Compute the PA_SC_AA_SAMPLE_LOCS_PIXEL_* mask based on hardware sample
 * locations.
 */
static void
radv_compute_sample_locs_pixel(uint32_t num_samples, VkOffset2D *sample_locs,
                               uint32_t *sample_locs_pixel)
{
   for (uint32_t i = 0; i < num_samples; i++) {
      uint32_t sample_reg_idx = i / 4;
      uint32_t sample_loc_idx = i % 4;
      int32_t pos_x = sample_locs[i].x;
      int32_t pos_y = sample_locs[i].y;

      uint32_t shift_x = 8 * sample_loc_idx;
      uint32_t shift_y = shift_x + 4;

      sample_locs_pixel[sample_reg_idx] |= (pos_x & 0xf) << shift_x;
      sample_locs_pixel[sample_reg_idx] |= (pos_y & 0xf) << shift_y;
   }
}

/**
 * Compute the PA_SC_CENTROID_PRIORITY_* mask based on the top left hardware
 * sample locations.
 */
static uint64_t
radv_compute_centroid_priority(struct radv_cmd_buffer *cmd_buffer, VkOffset2D *sample_locs,
                               uint32_t num_samples)
{
   uint32_t *centroid_priorities = alloca(num_samples * sizeof(*centroid_priorities));
   uint32_t sample_mask = num_samples - 1;
   uint32_t *distances = alloca(num_samples * sizeof(*distances));
   uint64_t centroid_priority = 0;

   /* Compute the distances from center for each sample. */
   for (int i = 0; i < num_samples; i++) {
      distances[i] = (sample_locs[i].x * sample_locs[i].x) + (sample_locs[i].y * sample_locs[i].y);
   }

   /* Compute the centroid priorities by looking at the distances array. */
   for (int i = 0; i < num_samples; i++) {
      uint32_t min_idx = 0;

      for (int j = 1; j < num_samples; j++) {
         if (distances[j] < distances[min_idx])
            min_idx = j;
      }

      centroid_priorities[i] = min_idx;
      distances[min_idx] = 0xffffffff;
   }

   /* Compute the final centroid priority. */
   for (int i = 0; i < 8; i++) {
      centroid_priority |= centroid_priorities[i & sample_mask] << (i * 4);
   }

   return centroid_priority << 32 | centroid_priority;
}

/**
 * Emit the sample locations that are specified with VK_EXT_sample_locations.
 */
static void
radv_emit_sample_locations(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_sample_locations_state *sample_location = &cmd_buffer->state.dynamic.sample_location;
   struct radv_graphics_pipeline *pipeline = cmd_buffer->state.graphics_pipeline;
   uint32_t num_samples = (uint32_t)sample_location->per_pixel;
   unsigned pa_sc_aa_config = pipeline->ms.pa_sc_aa_config;
   struct radeon_cmdbuf *cs = cmd_buffer->cs;
   uint32_t sample_locs_pixel[4][2] = {0};
   VkOffset2D sample_locs[4][8]; /* 8 is the max. sample count supported */
   uint32_t max_sample_dist = 0;
   uint64_t centroid_priority;

   if (!cmd_buffer->state.dynamic.sample_location.count)
      return;

   /* Convert the user sample locations to hardware sample locations. */
   radv_convert_user_sample_locs(sample_location, 0, 0, sample_locs[0]);
   radv_convert_user_sample_locs(sample_location, 1, 0, sample_locs[1]);
   radv_convert_user_sample_locs(sample_location, 0, 1, sample_locs[2]);
   radv_convert_user_sample_locs(sample_location, 1, 1, sample_locs[3]);

   /* Compute the PA_SC_AA_SAMPLE_LOCS_PIXEL_* mask. */
   for (uint32_t i = 0; i < 4; i++) {
      radv_compute_sample_locs_pixel(num_samples, sample_locs[i], sample_locs_pixel[i]);
   }

   /* Compute the PA_SC_CENTROID_PRIORITY_* mask. */
   centroid_priority = radv_compute_centroid_priority(cmd_buffer, sample_locs[0], num_samples);

   /* Compute the maximum sample distance from the specified locations. */
   for (unsigned i = 0; i < 4; ++i) {
      for (uint32_t j = 0; j < num_samples; j++) {
         VkOffset2D offset = sample_locs[i][j];
         max_sample_dist = MAX2(max_sample_dist, MAX2(abs(offset.x), abs(offset.y)));
      }
   }

   /* Emit the specified user sample locations. */
   switch (num_samples) {
   case 2:
   case 4:
      radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0,
                             sample_locs_pixel[0][0]);
      radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0,
                             sample_locs_pixel[1][0]);
      radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0,
                             sample_locs_pixel[2][0]);
      radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0,
                             sample_locs_pixel[3][0]);
      break;
   case 8:
      radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0,
                             sample_locs_pixel[0][0]);
      radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0,
                             sample_locs_pixel[1][0]);
      radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0,
                             sample_locs_pixel[2][0]);
      radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0,
                             sample_locs_pixel[3][0]);
      radeon_set_context_reg(cs, R_028BFC_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1,
                             sample_locs_pixel[0][1]);
      radeon_set_context_reg(cs, R_028C0C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1,
                             sample_locs_pixel[1][1]);
      radeon_set_context_reg(cs, R_028C1C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1,
                             sample_locs_pixel[2][1]);
      radeon_set_context_reg(cs, R_028C2C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1,
                             sample_locs_pixel[3][1]);
      break;
   default:
      unreachable("invalid number of samples");
   }

   /* Emit the maximum sample distance and the centroid priority. */
   pa_sc_aa_config &= C_028BE0_MAX_SAMPLE_DIST;
   pa_sc_aa_config |= S_028BE0_MAX_SAMPLE_DIST(max_sample_dist);

   radeon_set_context_reg(cs, R_028BE0_PA_SC_AA_CONFIG, pa_sc_aa_config);

   radeon_set_context_reg_seq(cs, R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2);
   radeon_emit(cs, centroid_priority);
   radeon_emit(cs, centroid_priority >> 32);

   cmd_buffer->state.context_roll_without_scissor_emitted = true;
}

static void
radv_emit_inline_push_consts(struct radv_device *device, struct radeon_cmdbuf *cs,
                             struct radv_pipeline *pipeline, gl_shader_stage stage, int idx,
                             uint32_t *values)
{
   struct radv_userdata_info *loc = radv_lookup_user_sgpr(pipeline, stage, idx);
   uint32_t base_reg = pipeline->user_data_0[stage];
   if (loc->sgpr_idx == -1)
      return;

   radeon_check_space(device->ws, cs, 2 + loc->num_sgprs);

   radeon_set_sh_reg_seq(cs, base_reg + loc->sgpr_idx * 4, loc->num_sgprs);
   radeon_emit_array(cs, values, loc->num_sgprs);
}

static void
radv_update_multisample_state(struct radv_cmd_buffer *cmd_buffer,
                              struct radv_graphics_pipeline *pipeline)
{
   int num_samples = pipeline->ms.num_samples;
   struct radv_graphics_pipeline *old_pipeline = cmd_buffer->state.emitted_graphics_pipeline;

   if (pipeline->base.shaders[MESA_SHADER_FRAGMENT]->info.ps.needs_sample_positions)
      cmd_buffer->sample_positions_needed = true;

   if (old_pipeline && num_samples == old_pipeline->ms.num_samples)
      return;

   radv_emit_default_sample_locations(cmd_buffer->cs, num_samples);

   cmd_buffer->state.context_roll_without_scissor_emitted = true;
}

static void
radv_update_binning_state(struct radv_cmd_buffer *cmd_buffer,
                          struct radv_graphics_pipeline *pipeline)
{
   const struct radv_graphics_pipeline *old_pipeline = cmd_buffer->state.emitted_graphics_pipeline;

   if (pipeline->base.device->physical_device->rad_info.gfx_level < GFX9)
      return;

   if (old_pipeline &&
       old_pipeline->binning.pa_sc_binner_cntl_0 ==
          pipeline->binning.pa_sc_binner_cntl_0)
      return;

   bool binning_flush = false;
   if (cmd_buffer->device->physical_device->rad_info.family == CHIP_VEGA12 ||
       cmd_buffer->device->physical_device->rad_info.family == CHIP_VEGA20 ||
       cmd_buffer->device->physical_device->rad_info.family == CHIP_RAVEN2 ||
       cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX10) {
      binning_flush = !old_pipeline ||
                      G_028C44_BINNING_MODE(old_pipeline->binning.pa_sc_binner_cntl_0) !=
                         G_028C44_BINNING_MODE(pipeline->binning.pa_sc_binner_cntl_0);
   }

   radeon_set_context_reg(cmd_buffer->cs, R_028C44_PA_SC_BINNER_CNTL_0,
                          pipeline->binning.pa_sc_binner_cntl_0 |
                             S_028C44_FLUSH_ON_BINNING_TRANSITION(!!binning_flush));

   cmd_buffer->state.context_roll_without_scissor_emitted = true;
}

static void
radv_emit_shader_prefetch(struct radv_cmd_buffer *cmd_buffer, struct radv_shader *shader)
{
   uint64_t va;

   if (!shader)
      return;

   va = radv_shader_get_va(shader);

   si_cp_dma_prefetch(cmd_buffer, va, shader->code_size);
}

static void
radv_emit_prefetch_L2(struct radv_cmd_buffer *cmd_buffer,
                      struct radv_graphics_pipeline *pipeline, bool first_stage_only)
{
   struct radv_cmd_state *state = &cmd_buffer->state;
   uint32_t mask = state->prefetch_L2_mask;

   /* Fast prefetch path for starting draws as soon as possible. */
   if (first_stage_only)
      mask &= RADV_PREFETCH_VS | RADV_PREFETCH_VBO_DESCRIPTORS | RADV_PREFETCH_MS;

   if (mask & RADV_PREFETCH_VS)
      radv_emit_shader_prefetch(cmd_buffer, pipeline->base.shaders[MESA_SHADER_VERTEX]);

   if (mask & RADV_PREFETCH_MS)
      radv_emit_shader_prefetch(cmd_buffer, pipeline->base.shaders[MESA_SHADER_MESH]);

   if (mask & RADV_PREFETCH_VBO_DESCRIPTORS)
      si_cp_dma_prefetch(cmd_buffer, state->vb_va, pipeline->vb_desc_alloc_size);

   if (mask & RADV_PREFETCH_TCS)
      radv_emit_shader_prefetch(cmd_buffer, pipeline->base.shaders[MESA_SHADER_TESS_CTRL]);

   if (mask & RADV_PREFETCH_TES)
      radv_emit_shader_prefetch(cmd_buffer, pipeline->base.shaders[MESA_SHADER_TESS_EVAL]);

   if (mask & RADV_PREFETCH_GS) {
      radv_emit_shader_prefetch(cmd_buffer, pipeline->base.shaders[MESA_SHADER_GEOMETRY]);
      if (radv_pipeline_has_gs_copy_shader(&pipeline->base))
         radv_emit_shader_prefetch(cmd_buffer, pipeline->base.gs_copy_shader);
   }

   if (mask & RADV_PREFETCH_PS) {
      radv_emit_shader_prefetch(cmd_buffer, pipeline->base.shaders[MESA_SHADER_FRAGMENT]);
      if (pipeline->ps_epilog) {
         struct radv_shader_part *ps_epilog = pipeline->ps_epilog;

         si_cp_dma_prefetch(cmd_buffer, ps_epilog->va, ps_epilog->code_size);
      }
   }

   state->prefetch_L2_mask &= ~mask;
}

static void
radv_emit_rbplus_state(struct radv_cmd_buffer *cmd_buffer)
{
   if (!cmd_buffer->device->physical_device->rad_info.rbplus_allowed)
      return;

   struct radv_graphics_pipeline *pipeline = cmd_buffer->state.graphics_pipeline;
   struct radv_rendering_state *render = &cmd_buffer->state.render;

   unsigned sx_ps_downconvert = 0;
   unsigned sx_blend_opt_epsilon = 0;
   unsigned sx_blend_opt_control = 0;

   for (unsigned i = 0; i < render->color_att_count; i++) {
      unsigned format, swap;
      bool has_alpha, has_rgb;
      if (render->color_att[i].iview == NULL) {
         /* We don't set the DISABLE bits, because the HW can't have holes,
          * so the SPI color format is set to 32-bit 1-component. */
         sx_ps_downconvert |= V_028754_SX_RT_EXPORT_32_R << (i * 4);
         continue;
      }

      struct radv_color_buffer_info *cb = &render->color_att[i].cb;

      format = cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX11
                  ? G_028C70_FORMAT_GFX11(cb->cb_color_info)
                  : G_028C70_FORMAT_GFX6(cb->cb_color_info);
      swap = G_028C70_COMP_SWAP(cb->cb_color_info);
      has_alpha = cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX11
                     ? !G_028C74_FORCE_DST_ALPHA_1_GFX11(cb->cb_color_attrib)
                     : !G_028C74_FORCE_DST_ALPHA_1_GFX6(cb->cb_color_attrib);

      uint32_t spi_format = (pipeline->col_format_non_compacted >> (i * 4)) & 0xf;
      uint32_t colormask = (pipeline->cb_target_mask >> (i * 4)) & 0xf;

      if (format == V_028C70_COLOR_8 || format == V_028C70_COLOR_16 || format == V_028C70_COLOR_32)
         has_rgb = !has_alpha;
      else
         has_rgb = true;

      /* Check the colormask and export format. */
      if (!(colormask & 0x7))
         has_rgb = false;
      if (!(colormask & 0x8))
         has_alpha = false;

      if (spi_format == V_028714_SPI_SHADER_ZERO) {
         has_rgb = false;
         has_alpha = false;
      }

      /* The HW doesn't quite blend correctly with rgb9e5 if we disable the alpha
       * optimization, even though it has no alpha. */
      if (has_rgb && format == V_028C70_COLOR_5_9_9_9)
         has_alpha = true;

      /* Disable value checking for disabled channels. */
      if (!has_rgb)
         sx_blend_opt_control |= S_02875C_MRT0_COLOR_OPT_DISABLE(1) << (i * 4);
      if (!has_alpha)
         sx_blend_opt_control |= S_02875C_MRT0_ALPHA_OPT_DISABLE(1) << (i * 4);

      /* Enable down-conversion for 32bpp and smaller formats. */
      switch (format) {
      case V_028C70_COLOR_8:
      case V_028C70_COLOR_8_8:
      case V_028C70_COLOR_8_8_8_8:
         /* For 1 and 2-channel formats, use the superset thereof. */
         if (spi_format == V_028714_SPI_SHADER_FP16_ABGR ||
             spi_format == V_028714_SPI_SHADER_UINT16_ABGR ||
             spi_format == V_028714_SPI_SHADER_SINT16_ABGR) {
            sx_ps_downconvert |= V_028754_SX_RT_EXPORT_8_8_8_8 << (i * 4);

            if (G_028C70_NUMBER_TYPE(cb->cb_color_info) != V_028C70_NUMBER_SRGB)
               sx_blend_opt_epsilon |= V_028758_8BIT_FORMAT << (i * 4);
         }
         break;

      case V_028C70_COLOR_5_6_5:
         if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) {
            sx_ps_downconvert |= V_028754_SX_RT_EXPORT_5_6_5 << (i * 4);
            sx_blend_opt_epsilon |= V_028758_6BIT_FORMAT << (i * 4);
         }
         break;

      case V_028C70_COLOR_1_5_5_5:
         if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) {
            sx_ps_downconvert |= V_028754_SX_RT_EXPORT_1_5_5_5 << (i * 4);
            sx_blend_opt_epsilon |= V_028758_5BIT_FORMAT << (i * 4);
         }
         break;

      case V_028C70_COLOR_4_4_4_4:
         if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) {
            sx_ps_downconvert |= V_028754_SX_RT_EXPORT_4_4_4_4 << (i * 4);
            sx_blend_opt_epsilon |= V_028758_4BIT_FORMAT << (i * 4);
         }
         break;

      case V_028C70_COLOR_32:
         if (swap == V_028C70_SWAP_STD && spi_format == V_028714_SPI_SHADER_32_R)
            sx_ps_downconvert |= V_028754_SX_RT_EXPORT_32_R << (i * 4);
         else if (swap == V_028C70_SWAP_ALT_REV && spi_format == V_028714_SPI_SHADER_32_AR)
            sx_ps_downconvert |= V_028754_SX_RT_EXPORT_32_A << (i * 4);
         break;

      case V_028C70_COLOR_16:
      case V_028C70_COLOR_16_16:
         /* For 1-channel formats, use the superset thereof. */
         if (spi_format == V_028714_SPI_SHADER_UNORM16_ABGR ||
             spi_format == V_028714_SPI_SHADER_SNORM16_ABGR ||
             spi_format == V_028714_SPI_SHADER_UINT16_ABGR ||
             spi_format == V_028714_SPI_SHADER_SINT16_ABGR) {
            if (swap == V_028C70_SWAP_STD || swap == V_028C70_SWAP_STD_REV)
               sx_ps_downconvert |= V_028754_SX_RT_EXPORT_16_16_GR << (i * 4);
            else
               sx_ps_downconvert |= V_028754_SX_RT_EXPORT_16_16_AR << (i * 4);
         }
         break;

      case V_028C70_COLOR_10_11_11:
         if (spi_format == V_028714_SPI_SHADER_FP16_ABGR)
            sx_ps_downconvert |= V_028754_SX_RT_EXPORT_10_11_11 << (i * 4);
         break;

      case V_028C70_COLOR_2_10_10_10:
         if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) {
            sx_ps_downconvert |= V_028754_SX_RT_EXPORT_2_10_10_10 << (i * 4);
            sx_blend_opt_epsilon |= V_028758_10BIT_FORMAT << (i * 4);
         }
         break;
      case V_028C70_COLOR_5_9_9_9:
         if (spi_format == V_028714_SPI_SHADER_FP16_ABGR)
            sx_ps_downconvert |= V_028754_SX_RT_EXPORT_9_9_9_E5 << (i * 4);
         break;
      }
   }

   /* Do not set the DISABLE bits for the unused attachments, as that
    * breaks dual source blending in SkQP and does not seem to improve
    * performance. */

   if (sx_ps_downconvert == cmd_buffer->state.last_sx_ps_downconvert &&
       sx_blend_opt_epsilon == cmd_buffer->state.last_sx_blend_opt_epsilon &&
       sx_blend_opt_control == cmd_buffer->state.last_sx_blend_opt_control)
      return;

   radeon_set_context_reg_seq(cmd_buffer->cs, R_028754_SX_PS_DOWNCONVERT, 3);
   radeon_emit(cmd_buffer->cs, sx_ps_downconvert);
   radeon_emit(cmd_buffer->cs, sx_blend_opt_epsilon);
   radeon_emit(cmd_buffer->cs, sx_blend_opt_control);

   cmd_buffer->state.context_roll_without_scissor_emitted = true;

   cmd_buffer->state.last_sx_ps_downconvert = sx_ps_downconvert;
   cmd_buffer->state.last_sx_blend_opt_epsilon = sx_blend_opt_epsilon;
   cmd_buffer->state.last_sx_blend_opt_control = sx_blend_opt_control;
}

static void
radv_emit_ps_epilog(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_graphics_pipeline *pipeline = cmd_buffer->state.graphics_pipeline;
   struct radv_shader *ps_shader = pipeline->base.shaders[MESA_SHADER_FRAGMENT];
   struct radv_shader_part *ps_epilog = pipeline->ps_epilog;

   if (!ps_epilog)
      return;

   /* The main shader must not use less VGPRs than the epilog, otherwise shared vgprs might not
    * work.
    */
   assert(G_00B848_VGPRS(ps_shader->config.rsrc1) >= G_00B848_VGPRS(ps_epilog->rsrc1));

   radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, ps_epilog->bo);

   assert((ps_epilog->va >> 32) == cmd_buffer->device->physical_device->rad_info.address32_hi);

   struct radv_userdata_info *loc =
      &ps_shader->info.user_sgprs_locs.shader_data[AC_UD_PS_EPILOG_PC];
   uint32_t base_reg = pipeline->base.user_data_0[MESA_SHADER_FRAGMENT];
   assert(loc->sgpr_idx != -1);
   assert(loc->num_sgprs == 1);
   radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, base_reg + loc->sgpr_idx * 4,
                            ps_epilog->va, false);
}

static void
radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_graphics_pipeline *pipeline = cmd_buffer->state.graphics_pipeline;
   const struct radv_device *device = cmd_buffer->device;

   if (cmd_buffer->state.emitted_graphics_pipeline == pipeline)
      return;

   radv_update_multisample_state(cmd_buffer, pipeline);
   radv_update_binning_state(cmd_buffer, pipeline);

   cmd_buffer->scratch_size_per_wave_needed =
      MAX2(cmd_buffer->scratch_size_per_wave_needed, pipeline->base.scratch_bytes_per_wave);
   cmd_buffer->scratch_waves_wanted = MAX2(cmd_buffer->scratch_waves_wanted, pipeline->base.max_waves);

   if (!cmd_buffer->state.emitted_graphics_pipeline)
      cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_PRIMITIVE_TOPOLOGY |
                                 RADV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS |
                                 RADV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS |
                                 RADV_CMD_DIRTY_DYNAMIC_PRIMITIVE_RESTART_ENABLE |
                                 RADV_CMD_DIRTY_DYNAMIC_DEPTH_TEST_ENABLE |
                                 RADV_CMD_DIRTY_DYNAMIC_DEPTH_WRITE_ENABLE |
                                 RADV_CMD_DIRTY_DYNAMIC_DEPTH_COMPARE_OP |
                                 RADV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS_TEST_ENABLE |
                                 RADV_CMD_DIRTY_DYNAMIC_STENCIL_TEST_ENABLE |
                                 RADV_CMD_DIRTY_DYNAMIC_STENCIL_OP |
                                 RADV_CMD_DIRTY_DYNAMIC_PATCH_CONTROL_POINTS |
                                 RADV_CMD_DIRTY_DYNAMIC_ALPHA_TO_COVERAGE_ENABLE |
                                 RADV_CMD_DIRTY_DYNAMIC_RASTERIZER_DISCARD_ENABLE |
                                 RADV_CMD_DIRTY_DYNAMIC_DEPTH_CLIP_ENABLE |
                                 RADV_CMD_DIRTY_DYNAMIC_DEPTH_CLIP_NEGATIVE_ONE_TO_ONE |
                                 RADV_CMD_DIRTY_DYNAMIC_CULL_MODE |
                                 RADV_CMD_DIRTY_DYNAMIC_FRONT_FACE |
                                 RADV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS |
                                 RADV_CMD_DIRTY_DYNAMIC_POLYGON_MODE |
                                 RADV_CMD_DIRTY_DYNAMIC_PROVOKING_VERTEX_MODE |
                                 RADV_CMD_DIRTY_DYNAMIC_VIEWPORT |
                                 RADV_CMD_DIRTY_DYNAMIC_DEPTH_CLAMP_ENABLE;

   if (!cmd_buffer->state.emitted_graphics_pipeline ||
       radv_rast_prim_is_points_or_lines(cmd_buffer->state.emitted_graphics_pipeline->rast_prim) != radv_rast_prim_is_points_or_lines(pipeline->rast_prim))
      cmd_buffer->state.dirty |= RADV_CMD_DIRTY_GUARDBAND;

   if (!cmd_buffer->state.emitted_graphics_pipeline ||
       cmd_buffer->state.emitted_graphics_pipeline->cb_color_control != pipeline->cb_color_control)
      cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_LOGIC_OP |
                                 RADV_CMD_DIRTY_DYNAMIC_LOGIC_OP_ENABLE;

   if (!cmd_buffer->state.emitted_graphics_pipeline ||
       cmd_buffer->state.emitted_graphics_pipeline->cb_target_mask != pipeline->cb_target_mask)
      cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_COLOR_WRITE_ENABLE;

   if (!cmd_buffer->state.emitted_graphics_pipeline ||
       cmd_buffer->state.emitted_graphics_pipeline->vgt_tf_param != pipeline->vgt_tf_param)
      cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_TESS_DOMAIN_ORIGIN;

   if (!cmd_buffer->state.emitted_graphics_pipeline ||
       cmd_buffer->state.emitted_graphics_pipeline->ms.pa_sc_mode_cntl_0 != pipeline->ms.pa_sc_mode_cntl_0)
      cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_LINE_STIPPLE_ENABLE;

   if (!cmd_buffer->state.emitted_graphics_pipeline ||
       cmd_buffer->state.emitted_graphics_pipeline->ms.pa_sc_aa_config != pipeline->ms.pa_sc_aa_config ||
       cmd_buffer->state.emitted_graphics_pipeline->ms.db_eqaa != pipeline->ms.db_eqaa)
      cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_CONSERVATIVE_RAST_MODE;

   radeon_emit_array(cmd_buffer->cs, pipeline->base.cs.buf, pipeline->base.cs.cdw);

   if (pipeline->has_ngg_culling &&
       pipeline->last_vgt_api_stage != MESA_SHADER_GEOMETRY &&
       !cmd_buffer->state.last_nggc_settings) {
      /* The already emitted RSRC2 contains the LDS required for NGG culling.
       * Culling is currently disabled, so re-emit RSRC2 to reduce LDS usage.
       * API GS always needs LDS, so this isn't useful there.
       */
      struct radv_shader *v = pipeline->base.shaders[pipeline->last_vgt_api_stage];
      radeon_set_sh_reg(cmd_buffer->cs, R_00B22C_SPI_SHADER_PGM_RSRC2_GS,
                        (v->config.rsrc2 & C_00B22C_LDS_SIZE) |
                        S_00B22C_LDS_SIZE(v->info.num_lds_blocks_when_not_culling));
   }

   if (!cmd_buffer->state.emitted_graphics_pipeline ||
       cmd_buffer->state.emitted_graphics_pipeline->base.ctx_cs.cdw != pipeline->base.ctx_cs.cdw ||
       cmd_buffer->state.emitted_graphics_pipeline->base.ctx_cs_hash != pipeline->base.ctx_cs_hash ||
       memcmp(cmd_buffer->state.emitted_graphics_pipeline->base.ctx_cs.buf, pipeline->base.ctx_cs.buf,
              pipeline->base.ctx_cs.cdw * 4)) {
      radeon_emit_array(cmd_buffer->cs, pipeline->base.ctx_cs.buf, pipeline->base.ctx_cs.cdw);
      cmd_buffer->state.context_roll_without_scissor_emitted = true;
   }

   if (device->pbb_allowed) {
      struct radv_binning_settings *settings = &device->physical_device->binning_settings;

      if ((!cmd_buffer->state.emitted_graphics_pipeline ||
           cmd_buffer->state.emitted_graphics_pipeline->base.shaders[MESA_SHADER_FRAGMENT] !=
           cmd_buffer->state.graphics_pipeline->base.shaders[MESA_SHADER_FRAGMENT]) &&
          (settings->context_states_per_bin > 1 || settings->persistent_states_per_bin > 1)) {
         /* Break the batch on PS changes. */
         radeon_emit(cmd_buffer->cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
         radeon_emit(cmd_buffer->cs, EVENT_TYPE(V_028A90_BREAK_BATCH) | EVENT_INDEX(0));
      }
   }

   radv_emit_ps_epilog(cmd_buffer);

   radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, pipeline->base.slab_bo);

   /* With graphics pipeline library, binaries are uploaded from a library and they hold a pointer
    * to the slab BO.
    */
   for (unsigned s = 0; s < MESA_VULKAN_SHADER_STAGES; s++) {
      struct radv_shader *shader = pipeline->base.shaders[s];

      if (!shader || !shader->bo)
         continue;

      radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, shader->bo);
   }

   if (pipeline->base.gs_copy_shader && pipeline->base.gs_copy_shader->bo) {
      radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, pipeline->base.gs_copy_shader->bo);
   }

   if (unlikely(cmd_buffer->device->trace_bo))
      radv_save_pipeline(cmd_buffer, &pipeline->base);

   cmd_buffer->state.emitted_graphics_pipeline = pipeline;

   cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_PIPELINE;
}

static enum radv_depth_clamp_mode
radv_get_depth_clamp_mode(struct radv_cmd_buffer *cmd_buffer)
{
   const struct radv_device *device = cmd_buffer->device;
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;
   enum radv_depth_clamp_mode mode;

   mode = RADV_DEPTH_CLAMP_MODE_VIEWPORT;
   if (!d->depth_clamp_enable) {
      /* For optimal performance, depth clamping should always be enabled except if the application
       * disables clamping explicitly or uses depth values outside of the [0.0, 1.0] range.
       */
      if (!d->depth_clip_enable || device->vk.enabled_extensions.EXT_depth_range_unrestricted) {
         mode = RADV_DEPTH_CLAMP_MODE_DISABLED;
      } else {
         mode = RADV_DEPTH_CLAMP_MODE_ZERO_TO_ONE;
      }
   }

   return mode;
}

static void
radv_emit_viewport(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;
   const struct radv_viewport_state *viewport = &cmd_buffer->state.dynamic.viewport;
   enum radv_depth_clamp_mode depth_clamp_mode = radv_get_depth_clamp_mode(cmd_buffer);
   int i;
   const unsigned count = viewport->count;

   assert(count);
   radeon_set_context_reg_seq(cmd_buffer->cs, R_02843C_PA_CL_VPORT_XSCALE, count * 6);

   for (i = 0; i < count; i++) {
      radeon_emit(cmd_buffer->cs, fui(viewport->xform[i].scale[0]));
      radeon_emit(cmd_buffer->cs, fui(viewport->xform[i].translate[0]));
      radeon_emit(cmd_buffer->cs, fui(viewport->xform[i].scale[1]));
      radeon_emit(cmd_buffer->cs, fui(viewport->xform[i].translate[1]));

      double scale_z, translate_z;
      if (d->depth_clip_negative_one_to_one) {
         scale_z = viewport->xform[i].scale[2] * 0.5f;
         translate_z = (viewport->xform[i].translate[2] + viewport->viewports[i].maxDepth) * 0.5f;
      } else {
         scale_z = viewport->xform[i].scale[2];
         translate_z = viewport->xform[i].translate[2];

      }
      radeon_emit(cmd_buffer->cs, fui(scale_z));
      radeon_emit(cmd_buffer->cs, fui(translate_z));
   }

   radeon_set_context_reg_seq(cmd_buffer->cs, R_0282D0_PA_SC_VPORT_ZMIN_0, count * 2);
   for (i = 0; i < count; i++) {
      float zmin, zmax;

      if (depth_clamp_mode == RADV_DEPTH_CLAMP_MODE_ZERO_TO_ONE) {
         zmin = 0.0f;
         zmax = 1.0f;
      } else {
         zmin = MIN2(viewport->viewports[i].minDepth, viewport->viewports[i].maxDepth);
         zmax = MAX2(viewport->viewports[i].minDepth, viewport->viewports[i].maxDepth);
      }

      radeon_emit(cmd_buffer->cs, fui(zmin));
      radeon_emit(cmd_buffer->cs, fui(zmax));
   }
}

void
radv_write_scissors(struct radv_cmd_buffer *cmd_buffer, struct radeon_cmdbuf *cs)
{
   uint32_t count = cmd_buffer->state.dynamic.scissor.count;

   si_write_scissors(cs, count, cmd_buffer->state.dynamic.scissor.scissors,
                     cmd_buffer->state.dynamic.viewport.viewports);
}

static void
radv_emit_scissor(struct radv_cmd_buffer *cmd_buffer)
{
   radv_write_scissors(cmd_buffer, cmd_buffer->cs);

   cmd_buffer->state.context_roll_without_scissor_emitted = false;
}

static void
radv_emit_discard_rectangle(struct radv_cmd_buffer *cmd_buffer)
{
   if (!cmd_buffer->state.dynamic.discard_rectangle.count)
      return;

   radeon_set_context_reg_seq(cmd_buffer->cs, R_028210_PA_SC_CLIPRECT_0_TL,
                              cmd_buffer->state.dynamic.discard_rectangle.count * 2);
   for (unsigned i = 0; i < cmd_buffer->state.dynamic.discard_rectangle.count; ++i) {
      VkRect2D rect = cmd_buffer->state.dynamic.discard_rectangle.rectangles[i];
      radeon_emit(cmd_buffer->cs, S_028210_TL_X(rect.offset.x) | S_028210_TL_Y(rect.offset.y));
      radeon_emit(cmd_buffer->cs, S_028214_BR_X(rect.offset.x + rect.extent.width) |
                                     S_028214_BR_Y(rect.offset.y + rect.extent.height));
   }
}

static void
radv_emit_line_width(struct radv_cmd_buffer *cmd_buffer)
{
   unsigned width = cmd_buffer->state.dynamic.line_width * 8;

   radeon_set_context_reg(cmd_buffer->cs, R_028A08_PA_SU_LINE_CNTL,
                          S_028A08_WIDTH(CLAMP(width, 0, 0xFFFF)));
}

static void
radv_emit_blend_constants(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;

   radeon_set_context_reg_seq(cmd_buffer->cs, R_028414_CB_BLEND_RED, 4);
   radeon_emit_array(cmd_buffer->cs, (uint32_t *)d->blend_constants, 4);
}

static void
radv_emit_stencil(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;

   radeon_set_context_reg_seq(cmd_buffer->cs, R_028430_DB_STENCILREFMASK, 2);
   radeon_emit(cmd_buffer->cs, S_028430_STENCILTESTVAL(d->stencil_reference.front) |
                                  S_028430_STENCILMASK(d->stencil_compare_mask.front) |
                                  S_028430_STENCILWRITEMASK(d->stencil_write_mask.front) |
                                  S_028430_STENCILOPVAL(1));
   radeon_emit(cmd_buffer->cs, S_028434_STENCILTESTVAL_BF(d->stencil_reference.back) |
                                  S_028434_STENCILMASK_BF(d->stencil_compare_mask.back) |
                                  S_028434_STENCILWRITEMASK_BF(d->stencil_write_mask.back) |
                                  S_028434_STENCILOPVAL_BF(1));
}

static void
radv_emit_depth_bounds(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;

   radeon_set_context_reg_seq(cmd_buffer->cs, R_028020_DB_DEPTH_BOUNDS_MIN, 2);
   radeon_emit(cmd_buffer->cs, fui(d->depth_bounds.min));
   radeon_emit(cmd_buffer->cs, fui(d->depth_bounds.max));
}

static void
radv_emit_depth_bias(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;
   unsigned slope = fui(d->depth_bias.slope * 16.0f);

   radeon_set_context_reg_seq(cmd_buffer->cs, R_028B7C_PA_SU_POLY_OFFSET_CLAMP, 5);
   radeon_emit(cmd_buffer->cs, fui(d->depth_bias.clamp)); /* CLAMP */
   radeon_emit(cmd_buffer->cs, slope);                    /* FRONT SCALE */
   radeon_emit(cmd_buffer->cs, fui(d->depth_bias.bias));  /* FRONT OFFSET */
   radeon_emit(cmd_buffer->cs, slope);                    /* BACK SCALE */
   radeon_emit(cmd_buffer->cs, fui(d->depth_bias.bias));  /* BACK OFFSET */
}

static void
radv_emit_line_stipple(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;
   uint32_t auto_reset_cntl = 1;

   if (d->primitive_topology == V_008958_DI_PT_LINESTRIP)
      auto_reset_cntl = 2;

   radeon_set_context_reg(cmd_buffer->cs, R_028A0C_PA_SC_LINE_STIPPLE,
                          S_028A0C_LINE_PATTERN(d->line_stipple.pattern) |
                             S_028A0C_REPEAT_COUNT(d->line_stipple.factor - 1) |
                             S_028A0C_AUTO_RESET_CNTL(auto_reset_cntl));
}

uint32_t
radv_get_pa_su_sc_mode_cntl(const struct radv_cmd_buffer *cmd_buffer)
{
   enum amd_gfx_level gfx_level = cmd_buffer->device->physical_device->rad_info.gfx_level;
   const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;
   unsigned pa_su_sc_mode_cntl;

   pa_su_sc_mode_cntl = S_028814_CULL_FRONT(!!(d->cull_mode & VK_CULL_MODE_FRONT_BIT)) |
                        S_028814_CULL_BACK(!!(d->cull_mode & VK_CULL_MODE_BACK_BIT)) |
                        S_028814_FACE(d->front_face) |
                        S_028814_POLY_OFFSET_FRONT_ENABLE(d->depth_bias_enable) |
                        S_028814_POLY_OFFSET_BACK_ENABLE(d->depth_bias_enable) |
                        S_028814_POLY_OFFSET_PARA_ENABLE(d->depth_bias_enable) |
                        S_028814_POLY_MODE(d->polygon_mode != V_028814_X_DRAW_TRIANGLES) |
                        S_028814_POLYMODE_FRONT_PTYPE(d->polygon_mode) |
                        S_028814_POLYMODE_BACK_PTYPE(d->polygon_mode) |
                        S_028814_PROVOKING_VTX_LAST(d->provoking_vertex_mode == VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT);

   if (gfx_level >= GFX10) {
      pa_su_sc_mode_cntl |=
         S_028814_KEEP_TOGETHER_ENABLE(d->polygon_mode != V_028814_X_DRAW_TRIANGLES);
   }

   return pa_su_sc_mode_cntl;
}

static void
radv_emit_culling(struct radv_cmd_buffer *cmd_buffer)
{
   unsigned pa_su_sc_mode_cntl = radv_get_pa_su_sc_mode_cntl(cmd_buffer);

   radeon_set_context_reg(cmd_buffer->cs, R_028814_PA_SU_SC_MODE_CNTL, pa_su_sc_mode_cntl);
}

static void
radv_emit_primitive_topology(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;

   assert(!cmd_buffer->state.mesh_shading);

   if (cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX7) {
      radeon_set_uconfig_reg_idx(cmd_buffer->device->physical_device, cmd_buffer->cs,
                                 R_030908_VGT_PRIMITIVE_TYPE, 1, d->primitive_topology);
   } else {
      radeon_set_config_reg(cmd_buffer->cs, R_008958_VGT_PRIMITIVE_TYPE, d->primitive_topology);
   }
}

static void
radv_emit_depth_control(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;

   radeon_set_context_reg(cmd_buffer->cs, R_028800_DB_DEPTH_CONTROL,
                          S_028800_Z_ENABLE(d->depth_test_enable ? 1 : 0) |
                          S_028800_Z_WRITE_ENABLE(d->depth_write_enable ? 1 : 0) |
                          S_028800_ZFUNC(d->depth_compare_op) |
                          S_028800_DEPTH_BOUNDS_ENABLE(d->depth_bounds_test_enable ? 1 : 0) |
                          S_028800_STENCIL_ENABLE(d->stencil_test_enable ? 1 : 0) |
                          S_028800_BACKFACE_ENABLE(d->stencil_test_enable ? 1 : 0) |
                          S_028800_STENCILFUNC(d->stencil_op.front.compare_op) |
                          S_028800_STENCILFUNC_BF(d->stencil_op.back.compare_op));
}

static void
radv_emit_stencil_control(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;

   radeon_set_context_reg(
      cmd_buffer->cs, R_02842C_DB_STENCIL_CONTROL,
      S_02842C_STENCILFAIL(si_translate_stencil_op(d->stencil_op.front.fail_op)) |
         S_02842C_STENCILZPASS(si_translate_stencil_op(d->stencil_op.front.pass_op)) |
         S_02842C_STENCILZFAIL(si_translate_stencil_op(d->stencil_op.front.depth_fail_op)) |
         S_02842C_STENCILFAIL_BF(si_translate_stencil_op(d->stencil_op.back.fail_op)) |
         S_02842C_STENCILZPASS_BF(si_translate_stencil_op(d->stencil_op.back.pass_op)) |
         S_02842C_STENCILZFAIL_BF(si_translate_stencil_op(d->stencil_op.back.depth_fail_op)));
}

static void
radv_emit_fragment_shading_rate(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_graphics_pipeline *pipeline = cmd_buffer->state.graphics_pipeline;
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;
   uint32_t rate_x = MIN2(2, d->fragment_shading_rate.size.width) - 1;
   uint32_t rate_y = MIN2(2, d->fragment_shading_rate.size.height) - 1;
   uint32_t pa_cl_vrs_cntl = pipeline->vrs.pa_cl_vrs_cntl;
   uint32_t pipeline_comb_mode = d->fragment_shading_rate.combiner_ops[0];
   uint32_t htile_comb_mode = d->fragment_shading_rate.combiner_ops[1];

   assert(cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX10_3);

   if (!cmd_buffer->state.render.vrs_att.iview) {
      /* When the current subpass has no VRS attachment, the VRS rates are expected to be 1x1, so we
       * can cheat by tweaking the different combiner modes.
       */
      switch (htile_comb_mode) {
      case VK_FRAGMENT_SHADING_RATE_COMBINER_OP_MIN_KHR:
         /* The result of min(A, 1x1) is always 1x1. */
         FALLTHROUGH;
      case VK_FRAGMENT_SHADING_RATE_COMBINER_OP_REPLACE_KHR:
         /* Force the per-draw VRS rate to 1x1. */
         rate_x = rate_y = 0;

         /* As the result of min(A, 1x1) or replace(A, 1x1) are always 1x1, set the vertex rate
          * combiner mode as passthrough.
          */
         pipeline_comb_mode = V_028848_VRS_COMB_MODE_PASSTHRU;
         break;
      case VK_FRAGMENT_SHADING_RATE_COMBINER_OP_MAX_KHR:
         /* The result of max(A, 1x1) is always A. */
         FALLTHROUGH;
      case VK_FRAGMENT_SHADING_RATE_COMBINER_OP_KEEP_KHR:
         /* Nothing to do here because the SAMPLE_ITER combiner mode should already be passthrough. */
         break;
      default:
         break;
      }
   }

   /* Emit per-draw VRS rate which is the first combiner. */
   radeon_set_uconfig_reg(cmd_buffer->cs, R_03098C_GE_VRS_RATE,
                          S_03098C_RATE_X(rate_x) | S_03098C_RATE_Y(rate_y));

   /* VERTEX_RATE_COMBINER_MODE controls the combiner mode between the
    * draw rate and the vertex rate.
    */
   if (cmd_buffer->state.mesh_shading) {
      pa_cl_vrs_cntl |= S_028848_VERTEX_RATE_COMBINER_MODE(V_028848_VRS_COMB_MODE_PASSTHRU) |
                        S_028848_PRIMITIVE_RATE_COMBINER_MODE(pipeline_comb_mode);
   } else {
      pa_cl_vrs_cntl |= S_028848_VERTEX_RATE_COMBINER_MODE(pipeline_comb_mode) |
                        S_028848_PRIMITIVE_RATE_COMBINER_MODE(V_028848_VRS_COMB_MODE_PASSTHRU);
   }

   /* HTILE_RATE_COMBINER_MODE controls the combiner mode between the primitive rate and the HTILE
    * rate.
    */
   pa_cl_vrs_cntl |= S_028848_HTILE_RATE_COMBINER_MODE(htile_comb_mode);

   radeon_set_context_reg(cmd_buffer->cs, R_028848_PA_CL_VRS_CNTL, pa_cl_vrs_cntl);
}

static void
radv_emit_primitive_restart_enable(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;

   if (cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX11) {
      radeon_set_uconfig_reg(cmd_buffer->cs, R_03092C_GE_MULTI_PRIM_IB_RESET_EN,
                             d->primitive_restart_enable);
   } else if (cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX9) {
      radeon_set_uconfig_reg(cmd_buffer->cs, R_03092C_VGT_MULTI_PRIM_IB_RESET_EN,
                             d->primitive_restart_enable);
   } else {
      radeon_set_context_reg(cmd_buffer->cs, R_028A94_VGT_MULTI_PRIM_IB_RESET_EN,
                             d->primitive_restart_enable);
   }
}

static void
radv_emit_clipping(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;

   radeon_set_context_reg(cmd_buffer->cs, R_028810_PA_CL_CLIP_CNTL,
                          S_028810_DX_RASTERIZATION_KILL(d->rasterizer_discard_enable) |
                          S_028810_ZCLIP_NEAR_DISABLE(!d->depth_clip_enable) |
                          S_028810_ZCLIP_FAR_DISABLE(!d->depth_clip_enable) |
                          S_028810_DX_CLIP_SPACE_DEF(!d->depth_clip_negative_one_to_one) |
                          S_028810_DX_LINEAR_ATTR_CLIP_ENA(1));
}

static void
radv_emit_logic_op(struct radv_cmd_buffer *cmd_buffer)
{
   unsigned cb_color_control = cmd_buffer->state.graphics_pipeline->cb_color_control;
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;

   if (d->logic_op_enable) {
      cb_color_control |= S_028808_ROP3(d->logic_op);
   } else {
      cb_color_control |= S_028808_ROP3(V_028808_ROP3_COPY);
   }

   if (cmd_buffer->device->physical_device->rad_info.has_rbplus) {
      cb_color_control |= S_028808_DISABLE_DUAL_QUAD(d->logic_op_enable);
   }

   radeon_set_context_reg(cmd_buffer->cs, R_028808_CB_COLOR_CONTROL, cb_color_control);
}

static void
radv_emit_color_write_enable(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_graphics_pipeline *pipeline = cmd_buffer->state.graphics_pipeline;
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;

   radeon_set_context_reg(cmd_buffer->cs, R_028238_CB_TARGET_MASK,
                          pipeline->cb_target_mask & d->color_write_enable);
}

static void
radv_emit_patch_control_points(struct radv_cmd_buffer *cmd_buffer)
{
   const struct radv_physical_device *pdevice = cmd_buffer->device->physical_device;
   struct radv_graphics_pipeline *pipeline = cmd_buffer->state.graphics_pipeline;
   struct radv_shader *tcs = pipeline->base.shaders[MESA_SHADER_TESS_CTRL];
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;
   unsigned ls_hs_config, base_reg;
   struct radv_userdata_info *loc;

   ls_hs_config = S_028B58_NUM_PATCHES(cmd_buffer->state.tess_num_patches) |
                  S_028B58_HS_NUM_INPUT_CP(d->patch_control_points) |
                  S_028B58_HS_NUM_OUTPUT_CP(tcs->info.tcs.tcs_vertices_out);

   if (pdevice->rad_info.gfx_level >= GFX7) {
      radeon_set_context_reg_idx(cmd_buffer->cs, R_028B58_VGT_LS_HS_CONFIG, 2, ls_hs_config);
   } else {
      radeon_set_context_reg(cmd_buffer->cs, R_028B58_VGT_LS_HS_CONFIG, ls_hs_config);
   }

   if (pdevice->rad_info.gfx_level >= GFX9) {
      unsigned hs_rsrc2 = tcs->config.rsrc2;

      if (pdevice->rad_info.gfx_level >= GFX10) {
         hs_rsrc2 |= S_00B42C_LDS_SIZE_GFX10(cmd_buffer->state.tess_lds_size);
      } else {
         hs_rsrc2 |= S_00B42C_LDS_SIZE_GFX9(cmd_buffer->state.tess_lds_size);
      }

      radeon_set_sh_reg(cmd_buffer->cs, R_00B42C_SPI_SHADER_PGM_RSRC2_HS, hs_rsrc2);
   } else {
      struct radv_shader *vs = pipeline->base.shaders[MESA_SHADER_VERTEX];
      unsigned ls_rsrc2 = vs->config.rsrc2 | S_00B52C_LDS_SIZE(cmd_buffer->state.tess_lds_size);

      radeon_set_sh_reg(cmd_buffer->cs, R_00B52C_SPI_SHADER_PGM_RSRC2_LS, ls_rsrc2);
   }

   /* Emit user SGPRs for dynamic patch control points. */
   loc = radv_lookup_user_sgpr(&pipeline->base, MESA_SHADER_TESS_CTRL, AC_UD_TCS_OFFCHIP_LAYOUT);
   if (loc->sgpr_idx == -1)
      return;
   assert(loc->num_sgprs == 1);

   base_reg = pipeline->base.user_data_0[MESA_SHADER_TESS_CTRL];
   radeon_set_sh_reg(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4,
                     (cmd_buffer->state.tess_num_patches << 6) | d->patch_control_points);

   loc = radv_lookup_user_sgpr(&pipeline->base, MESA_SHADER_TESS_EVAL, AC_UD_TES_NUM_PATCHES);
   assert(loc->sgpr_idx != -1 && loc->num_sgprs == 1);

   base_reg = pipeline->base.user_data_0[MESA_SHADER_TESS_EVAL];
   radeon_set_sh_reg(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4,
                     cmd_buffer->state.tess_num_patches);
}

static void
radv_emit_conservative_rast_mode(struct radv_cmd_buffer *cmd_buffer)
{
   const struct radv_physical_device *pdevice = cmd_buffer->device->physical_device;
   struct radv_graphics_pipeline *pipeline = cmd_buffer->state.graphics_pipeline;
   struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;
   unsigned pa_sc_aa_config = pipeline->ms.pa_sc_aa_config;
   unsigned db_eqaa = pipeline->ms.db_eqaa;

   if (pdevice->rad_info.gfx_level >= GFX9) {
      uint32_t pa_sc_conservative_rast = S_028C4C_NULL_SQUAD_AA_MASK_ENABLE(1);

      if (d->conservative_rast_mode != VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT) {
         pa_sc_conservative_rast = S_028C4C_PREZ_AA_MASK_ENABLE(1) | S_028C4C_POSTZ_AA_MASK_ENABLE(1) |
                                   S_028C4C_CENTROID_SAMPLE_OVERRIDE(1);

         if (d->conservative_rast_mode == VK_CONSERVATIVE_RASTERIZATION_MODE_OVERESTIMATE_EXT) {
            pa_sc_conservative_rast |=
               S_028C4C_OVER_RAST_ENABLE(1) | S_028C4C_OVER_RAST_SAMPLE_SELECT(0) |
               S_028C4C_UNDER_RAST_ENABLE(0) | S_028C4C_UNDER_RAST_SAMPLE_SELECT(1) |
               S_028C4C_PBB_UNCERTAINTY_REGION_ENABLE(1);
         } else {
            assert(d->conservative_rast_mode == VK_CONSERVATIVE_RASTERIZATION_MODE_UNDERESTIMATE_EXT);
            pa_sc_conservative_rast |=
               S_028C4C_OVER_RAST_ENABLE(0) | S_028C4C_OVER_RAST_SAMPLE_SELECT(1) |
               S_028C4C_UNDER_RAST_ENABLE(1) | S_028C4C_UNDER_RAST_SAMPLE_SELECT(0) |
               S_028C4C_PBB_UNCERTAINTY_REGION_ENABLE(0);
         }

         /* Adjust MSAA state if conservative rasterization is enabled. */
         pa_sc_aa_config |= S_028BE0_AA_MASK_CENTROID_DTMN(1);
         db_eqaa |= S_028804_ENABLE_POSTZ_OVERRASTERIZATION(1) |
                    S_028804_OVERRASTERIZATION_AMOUNT(4);
      }

      radeon_set_context_reg(cmd_buffer->cs, R_028C4C_PA_SC_CONSERVATIVE_RASTERIZATION_CNTL,
                             pa_sc_conservative_rast);
   }

   radeon_set_context_reg(cmd_buffer->cs, R_028BE0_PA_SC_AA_CONFIG, pa_sc_aa_config);
   radeon_set_context_reg(cmd_buffer->cs, R_028804_DB_EQAA, db_eqaa);
}

static void
radv_emit_depth_clamp_enable(struct radv_cmd_buffer *cmd_buffer)
{
   enum radv_depth_clamp_mode mode = radv_get_depth_clamp_mode(cmd_buffer);

   radeon_set_context_reg(cmd_buffer->cs, R_02800C_DB_RENDER_OVERRIDE,
                          S_02800C_FORCE_HIS_ENABLE0(V_02800C_FORCE_DISABLE) |
                          S_02800C_FORCE_HIS_ENABLE1(V_02800C_FORCE_DISABLE) |
                          S_02800C_DISABLE_VIEWPORT_CLAMP(mode == RADV_DEPTH_CLAMP_MODE_DISABLED));
}

static void
radv_emit_fb_color_state(struct radv_cmd_buffer *cmd_buffer, int index,
                         struct radv_color_buffer_info *cb, struct radv_image_view *iview,
                         VkImageLayout layout)
{
   bool is_vi = cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX8;
   uint32_t cb_fdcc_control = cb->cb_dcc_control;
   uint32_t cb_color_info = cb->cb_color_info;
   struct radv_image *image = iview->image;

   if (!radv_layout_dcc_compressed(
          cmd_buffer->device, image, iview->vk.base_mip_level, layout,
          radv_image_queue_family_mask(image, cmd_buffer->qf,
                                       cmd_buffer->qf))) {
      if (cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX11) {
         cb_fdcc_control &= C_028C78_FDCC_ENABLE;
      } else {
         cb_color_info &= C_028C70_DCC_ENABLE;
      }
   }

   if (!radv_layout_fmask_compressed(
          cmd_buffer->device, image, layout,
          radv_image_queue_family_mask(image, cmd_buffer->qf,
                                       cmd_buffer->qf))) {
      cb_color_info &= C_028C70_COMPRESSION;
   }

   if (radv_image_is_tc_compat_cmask(image) && (radv_is_fmask_decompress_pipeline(cmd_buffer) ||
                                                radv_is_dcc_decompress_pipeline(cmd_buffer))) {
      /* If this bit is set, the FMASK decompression operation
       * doesn't occur (DCC_COMPRESS also implies FMASK_DECOMPRESS).
       */
      cb_color_info &= C_028C70_FMASK_COMPRESS_1FRAG_ONLY;
   }

   if (cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX11) {
      radeon_set_context_reg_seq(cmd_buffer->cs, R_028C6C_CB_COLOR0_VIEW + index * 0x3c, 4);
      radeon_emit(cmd_buffer->cs, cb->cb_color_view);                      /* CB_COLOR0_VIEW */
      radeon_emit(cmd_buffer->cs, cb->cb_color_info);                      /* CB_COLOR0_INFO */
      radeon_emit(cmd_buffer->cs, cb->cb_color_attrib);                    /* CB_COLOR0_ATTRIB */
      radeon_emit(cmd_buffer->cs, cb_fdcc_control);                        /* CB_COLOR0_FDCC_CONTROL */

      radeon_set_context_reg(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, cb->cb_color_base);
      radeon_set_context_reg(cmd_buffer->cs, R_028E40_CB_COLOR0_BASE_EXT + index * 4, cb->cb_color_base >> 32);
      radeon_set_context_reg(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->cb_dcc_base);
      radeon_set_context_reg(cmd_buffer->cs, R_028EA0_CB_COLOR0_DCC_BASE_EXT + index * 4, cb->cb_dcc_base >> 32);
      radeon_set_context_reg(cmd_buffer->cs, R_028EC0_CB_COLOR0_ATTRIB2 + index * 4, cb->cb_color_attrib2);
      radeon_set_context_reg(cmd_buffer->cs, R_028EE0_CB_COLOR0_ATTRIB3 + index * 4, cb->cb_color_attrib3);
   } else if (cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX10) {
      radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 11);
      radeon_emit(cmd_buffer->cs, cb->cb_color_base);
      radeon_emit(cmd_buffer->cs, 0);
      radeon_emit(cmd_buffer->cs, 0);
      radeon_emit(cmd_buffer->cs, cb->cb_color_view);
      radeon_emit(cmd_buffer->cs, cb_color_info);
      radeon_emit(cmd_buffer->cs, cb->cb_color_attrib);
      radeon_emit(cmd_buffer->cs, cb->cb_dcc_control);
      radeon_emit(cmd_buffer->cs, cb->cb_color_cmask);
      radeon_emit(cmd_buffer->cs, 0);
      radeon_emit(cmd_buffer->cs, cb->cb_color_fmask);
      radeon_emit(cmd_buffer->cs, 0);

      radeon_set_context_reg(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->cb_dcc_base);

      radeon_set_context_reg(cmd_buffer->cs, R_028E40_CB_COLOR0_BASE_EXT + index * 4,
                             cb->cb_color_base >> 32);
      radeon_set_context_reg(cmd_buffer->cs, R_028E60_CB_COLOR0_CMASK_BASE_EXT + index * 4,
                             cb->cb_color_cmask >> 32);
      radeon_set_context_reg(cmd_buffer->cs, R_028E80_CB_COLOR0_FMASK_BASE_EXT + index * 4,
                             cb->cb_color_fmask >> 32);
      radeon_set_context_reg(cmd_buffer->cs, R_028EA0_CB_COLOR0_DCC_BASE_EXT + index * 4,
                             cb->cb_dcc_base >> 32);
      radeon_set_context_reg(cmd_buffer->cs, R_028EC0_CB_COLOR0_ATTRIB2 + index * 4,
                             cb->cb_color_attrib2);
      radeon_set_context_reg(cmd_buffer->cs, R_028EE0_CB_COLOR0_ATTRIB3 + index * 4,
                             cb->cb_color_attrib3);
   } else if (cmd_buffer->device->physical_device->rad_info.gfx_level == GFX9) {
      radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 11);
      radeon_emit(cmd_buffer->cs, cb->cb_color_base);
      radeon_emit(cmd_buffer->cs, S_028C64_BASE_256B(cb->cb_color_base >> 32));
      radeon_emit(cmd_buffer->cs, cb->cb_color_attrib2);
      radeon_emit(cmd_buffer->cs, cb->cb_color_view);
      radeon_emit(cmd_buffer->cs, cb_color_info);
      radeon_emit(cmd_buffer->cs, cb->cb_color_attrib);
      radeon_emit(cmd_buffer->cs, cb->cb_dcc_control);
      radeon_emit(cmd_buffer->cs, cb->cb_color_cmask);
      radeon_emit(cmd_buffer->cs, S_028C80_BASE_256B(cb->cb_color_cmask >> 32));
      radeon_emit(cmd_buffer->cs, cb->cb_color_fmask);
      radeon_emit(cmd_buffer->cs, S_028C88_BASE_256B(cb->cb_color_fmask >> 32));

      radeon_set_context_reg_seq(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, 2);
      radeon_emit(cmd_buffer->cs, cb->cb_dcc_base);
      radeon_emit(cmd_buffer->cs, S_028C98_BASE_256B(cb->cb_dcc_base >> 32));

      radeon_set_context_reg(cmd_buffer->cs, R_0287A0_CB_MRT0_EPITCH + index * 4,
                             cb->cb_mrt_epitch);
   } else {
      radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 11);
      radeon_emit(cmd_buffer->cs, cb->cb_color_base);
      radeon_emit(cmd_buffer->cs, cb->cb_color_pitch);
      radeon_emit(cmd_buffer->cs, cb->cb_color_slice);
      radeon_emit(cmd_buffer->cs, cb->cb_color_view);
      radeon_emit(cmd_buffer->cs, cb_color_info);
      radeon_emit(cmd_buffer->cs, cb->cb_color_attrib);
      radeon_emit(cmd_buffer->cs, cb->cb_dcc_control);
      radeon_emit(cmd_buffer->cs, cb->cb_color_cmask);
      radeon_emit(cmd_buffer->cs, cb->cb_color_cmask_slice);
      radeon_emit(cmd_buffer->cs, cb->cb_color_fmask);
      radeon_emit(cmd_buffer->cs, cb->cb_color_fmask_slice);

      if (is_vi) { /* DCC BASE */
         radeon_set_context_reg(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c,
                                cb->cb_dcc_base);
      }
   }

   if (cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX11
          ? G_028C78_FDCC_ENABLE(cb_fdcc_control)
          : G_028C70_DCC_ENABLE(cb_color_info)) {
      /* Drawing with DCC enabled also compresses colorbuffers. */
      VkImageSubresourceRange range = {
         .aspectMask = iview->vk.aspects,
         .baseMipLevel = iview->vk.base_mip_level,
         .levelCount = iview->vk.level_count,
         .baseArrayLayer = iview->vk.base_array_layer,
         .layerCount = iview->vk.layer_count,
      };

      radv_update_dcc_metadata(cmd_buffer, image, &range, true);
   }
}

static void
radv_update_zrange_precision(struct radv_cmd_buffer *cmd_buffer, struct radv_ds_buffer_info *ds,
                             const struct radv_image_view *iview, VkImageLayout layout,
                             bool requires_cond_exec)
{
   const struct radv_image *image = iview->image;
   uint32_t db_z_info = ds->db_z_info;
   uint32_t db_z_info_reg;

   if (!cmd_buffer->device->physical_device->rad_info.has_tc_compat_zrange_bug ||
       !radv_image_is_tc_compat_htile(image))
      return;

   if (!radv_layout_is_htile_compressed(
          cmd_buffer->device, image, layout,
          radv_image_queue_family_mask(image, cmd_buffer->qf,
                                       cmd_buffer->qf))) {
      db_z_info &= C_028040_TILE_SURFACE_ENABLE;
   }

   db_z_info &= C_028040_ZRANGE_PRECISION;

   if (cmd_buffer->device->physical_device->rad_info.gfx_level == GFX9) {
      db_z_info_reg = R_028038_DB_Z_INFO;
   } else {
      db_z_info_reg = R_028040_DB_Z_INFO;
   }

   /* When we don't know the last fast clear value we need to emit a
    * conditional packet that will eventually skip the following
    * SET_CONTEXT_REG packet.
    */
   if (requires_cond_exec) {
      uint64_t va = radv_get_tc_compat_zrange_va(image, iview->vk.base_mip_level);

      radeon_emit(cmd_buffer->cs, PKT3(PKT3_COND_EXEC, 3, 0));
      radeon_emit(cmd_buffer->cs, va);
      radeon_emit(cmd_buffer->cs, va >> 32);
      radeon_emit(cmd_buffer->cs, 0);
      radeon_emit(cmd_buffer->cs, 3); /* SET_CONTEXT_REG size */
   }

   radeon_set_context_reg(cmd_buffer->cs, db_z_info_reg, db_z_info);
}

static void
radv_emit_fb_ds_state(struct radv_cmd_buffer *cmd_buffer, struct radv_ds_buffer_info *ds,
                      struct radv_image_view *iview, VkImageLayout layout)
{
   const struct radv_image *image = iview->image;
   uint32_t db_z_info = ds->db_z_info;
   uint32_t db_stencil_info = ds->db_stencil_info;
   uint32_t db_htile_surface = ds->db_htile_surface;

   if (!radv_layout_is_htile_compressed(
          cmd_buffer->device, image, layout,
          radv_image_queue_family_mask(image, cmd_buffer->qf,
                                       cmd_buffer->qf))) {
      db_z_info &= C_028040_TILE_SURFACE_ENABLE;
      db_stencil_info |= S_028044_TILE_STENCIL_DISABLE(1);
   }

   if (cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX10_3 &&
       !cmd_buffer->state.render.vrs_att.iview) {
      db_htile_surface &= C_028ABC_VRS_HTILE_ENCODING;
   }

   radeon_set_context_reg(cmd_buffer->cs, R_028008_DB_DEPTH_VIEW, ds->db_depth_view);
   radeon_set_context_reg(cmd_buffer->cs, R_028ABC_DB_HTILE_SURFACE, db_htile_surface);

   if (cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX10) {
      radeon_set_context_reg(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, ds->db_htile_data_base);
      radeon_set_context_reg(cmd_buffer->cs, R_02801C_DB_DEPTH_SIZE_XY, ds->db_depth_size);

      if (cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX11) {
         radeon_set_context_reg_seq(cmd_buffer->cs, R_028040_DB_Z_INFO, 6);
      } else {
         radeon_set_context_reg_seq(cmd_buffer->cs, R_02803C_DB_DEPTH_INFO, 7);
         radeon_emit(cmd_buffer->cs, S_02803C_RESOURCE_LEVEL(1));
      }
      radeon_emit(cmd_buffer->cs, db_z_info);
      radeon_emit(cmd_buffer->cs, db_stencil_info);
      radeon_emit(cmd_buffer->cs, ds->db_z_read_base);
      radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base);
      radeon_emit(cmd_buffer->cs, ds->db_z_read_base);
      radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base);

      radeon_set_context_reg_seq(cmd_buffer->cs, R_028068_DB_Z_READ_BASE_HI, 5);
      radeon_emit(cmd_buffer->cs, ds->db_z_read_base >> 32);
      radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base >> 32);
      radeon_emit(cmd_buffer->cs, ds->db_z_read_base >> 32);
      radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base >> 32);
      radeon_emit(cmd_buffer->cs, ds->db_htile_data_base >> 32);
   } else if (cmd_buffer->device->physical_device->rad_info.gfx_level == GFX9) {
      radeon_set_context_reg_seq(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, 3);
      radeon_emit(cmd_buffer->cs, ds->db_htile_data_base);
      radeon_emit(cmd_buffer->cs, S_028018_BASE_HI(ds->db_htile_data_base >> 32));
      radeon_emit(cmd_buffer->cs, ds->db_depth_size);

      radeon_set_context_reg_seq(cmd_buffer->cs, R_028038_DB_Z_INFO, 10);
      radeon_emit(cmd_buffer->cs, db_z_info);          /* DB_Z_INFO */
      radeon_emit(cmd_buffer->cs, db_stencil_info);    /* DB_STENCIL_INFO */
      radeon_emit(cmd_buffer->cs, ds->db_z_read_base); /* DB_Z_READ_BASE */
      radeon_emit(cmd_buffer->cs,
                  S_028044_BASE_HI(ds->db_z_read_base >> 32)); /* DB_Z_READ_BASE_HI */
      radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base);   /* DB_STENCIL_READ_BASE */
      radeon_emit(cmd_buffer->cs,
                  S_02804C_BASE_HI(ds->db_stencil_read_base >> 32)); /* DB_STENCIL_READ_BASE_HI */
      radeon_emit(cmd_buffer->cs, ds->db_z_write_base);              /* DB_Z_WRITE_BASE */
      radeon_emit(cmd_buffer->cs,
                  S_028054_BASE_HI(ds->db_z_write_base >> 32)); /* DB_Z_WRITE_BASE_HI */
      radeon_emit(cmd_buffer->cs, ds->db_stencil_write_base);   /* DB_STENCIL_WRITE_BASE */
      radeon_emit(cmd_buffer->cs,
                  S_02805C_BASE_HI(ds->db_stencil_write_base >> 32)); /* DB_STENCIL_WRITE_BASE_HI */

      radeon_set_context_reg_seq(cmd_buffer->cs, R_028068_DB_Z_INFO2, 2);
      radeon_emit(cmd_buffer->cs, ds->db_z_info2);
      radeon_emit(cmd_buffer->cs, ds->db_stencil_info2);
   } else {
      radeon_set_context_reg(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, ds->db_htile_data_base);

      radeon_set_context_reg_seq(cmd_buffer->cs, R_02803C_DB_DEPTH_INFO, 9);
      radeon_emit(cmd_buffer->cs, ds->db_depth_info);         /* R_02803C_DB_DEPTH_INFO */
      radeon_emit(cmd_buffer->cs, db_z_info);                 /* R_028040_DB_Z_INFO */
      radeon_emit(cmd_buffer->cs, db_stencil_info);           /* R_028044_DB_STENCIL_INFO */
      radeon_emit(cmd_buffer->cs, ds->db_z_read_base);        /* R_028048_DB_Z_READ_BASE */
      radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base);  /* R_02804C_DB_STENCIL_READ_BASE */
      radeon_emit(cmd_buffer->cs, ds->db_z_write_base);       /* R_028050_DB_Z_WRITE_BASE */
      radeon_emit(cmd_buffer->cs, ds->db_stencil_write_base); /* R_028054_DB_STENCIL_WRITE_BASE */
      radeon_emit(cmd_buffer->cs, ds->db_depth_size);         /* R_028058_DB_DEPTH_SIZE */
      radeon_emit(cmd_buffer->cs, ds->db_depth_slice);        /* R_02805C_DB_DEPTH_SLICE */
   }

   /* Update the ZRANGE_PRECISION value for the TC-compat bug. */
   radv_update_zrange_precision(cmd_buffer, ds, iview, layout, true);

   radeon_set_context_reg(cmd_buffer->cs, R_028B78_PA_SU_POLY_OFFSET_DB_FMT_CNTL,
                          ds->pa_su_poly_offset_db_fmt_cntl);
}

/**
 * Update the fast clear depth/stencil values if the image is bound as a
 * depth/stencil buffer.
 */
static void
radv_update_bound_fast_clear_ds(struct radv_cmd_buffer *cmd_buffer,
                                const struct radv_image_view *iview,
                                VkClearDepthStencilValue ds_clear_value, VkImageAspectFlags aspects)
{
   const struct radv_image *image = iview->image;
   struct radeon_cmdbuf *cs = cmd_buffer->cs;

   if (cmd_buffer->state.render.ds_att.iview == NULL ||
       cmd_buffer->state.render.ds_att.iview->image != image)
      return;

   if (aspects == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) {
      radeon_set_context_reg_seq(cs, R_028028_DB_STENCIL_CLEAR, 2);
      radeon_emit(cs, ds_clear_value.stencil);
      radeon_emit(cs, fui(ds_clear_value.depth));
   } else if (aspects == VK_IMAGE_ASPECT_DEPTH_BIT) {
      radeon_set_context_reg(cs, R_02802C_DB_DEPTH_CLEAR, fui(ds_clear_value.depth));
   } else {
      assert(aspects == VK_IMAGE_ASPECT_STENCIL_BIT);
      radeon_set_context_reg(cs, R_028028_DB_STENCIL_CLEAR, ds_clear_value.stencil);
   }

   /* Update the ZRANGE_PRECISION value for the TC-compat bug. This is
    * only needed when clearing Z to 0.0.
    */
   if ((aspects & VK_IMAGE_ASPECT_DEPTH_BIT) && ds_clear_value.depth == 0.0) {
      radv_update_zrange_precision(cmd_buffer, &cmd_buffer->state.render.ds_att.ds, iview,
                                   cmd_buffer->state.render.ds_att.layout, false);
   }

   cmd_buffer->state.context_roll_without_scissor_emitted = true;
}

/**
 * Set the clear depth/stencil values to the image's metadata.
 */
static void
radv_set_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
                           const VkImageSubresourceRange *range,
                           VkClearDepthStencilValue ds_clear_value, VkImageAspectFlags aspects)
{
   struct radeon_cmdbuf *cs = cmd_buffer->cs;
   uint32_t level_count = radv_get_levelCount(image, range);

   if (aspects == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) {
      uint64_t va = radv_get_ds_clear_value_va(image, range->baseMipLevel);

      /* Use the fastest way when both aspects are used. */
      radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + 2 * level_count, cmd_buffer->state.predicating));
      radeon_emit(cs, S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_PFP));
      radeon_emit(cs, va);
      radeon_emit(cs, va >> 32);

      for (uint32_t l = 0; l < level_count; l++) {
         radeon_emit(cs, ds_clear_value.stencil);
         radeon_emit(cs, fui(ds_clear_value.depth));
      }
   } else {
      /* Otherwise we need one WRITE_DATA packet per level. */
      for (uint32_t l = 0; l < level_count; l++) {
         uint64_t va = radv_get_ds_clear_value_va(image, range->baseMipLevel + l);
         unsigned value;

         if (aspects == VK_IMAGE_ASPECT_DEPTH_BIT) {
            value = fui(ds_clear_value.depth);
            va += 4;
         } else {
            assert(aspects == VK_IMAGE_ASPECT_STENCIL_BIT);
            value = ds_clear_value.stencil;
         }

         radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, cmd_buffer->state.predicating));
         radeon_emit(cs,
                     S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_PFP));
         radeon_emit(cs, va);
         radeon_emit(cs, va >> 32);
         radeon_emit(cs, value);
      }
   }
}

/**
 * Update the TC-compat metadata value for this image.
 */
static void
radv_set_tc_compat_zrange_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
                                   const VkImageSubresourceRange *range, uint32_t value)
{
   struct radeon_cmdbuf *cs = cmd_buffer->cs;

   if (!cmd_buffer->device->physical_device->rad_info.has_tc_compat_zrange_bug)
      return;

   uint64_t va = radv_get_tc_compat_zrange_va(image, range->baseMipLevel);
   uint32_t level_count = radv_get_levelCount(image, range);

   radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + level_count, cmd_buffer->state.predicating));
   radeon_emit(cs, S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_PFP));
   radeon_emit(cs, va);
   radeon_emit(cs, va >> 32);

   for (uint32_t l = 0; l < level_count; l++)
      radeon_emit(cs, value);
}

static void
radv_update_tc_compat_zrange_metadata(struct radv_cmd_buffer *cmd_buffer,
                                      const struct radv_image_view *iview,
                                      VkClearDepthStencilValue ds_clear_value)
{
   VkImageSubresourceRange range = {
      .aspectMask = iview->vk.aspects,
      .baseMipLevel = iview->vk.base_mip_level,
      .levelCount = iview->vk.level_count,
      .baseArrayLayer = iview->vk.base_array_layer,
      .layerCount = iview->vk.layer_count,
   };
   uint32_t cond_val;

   /* Conditionally set DB_Z_INFO.ZRANGE_PRECISION to 0 when the last
    * depth clear value is 0.0f.
    */
   cond_val = ds_clear_value.depth == 0.0f ? UINT_MAX : 0;

   radv_set_tc_compat_zrange_metadata(cmd_buffer, iview->image, &range, cond_val);
}

/**
 * Update the clear depth/stencil values for this image.
 */
void
radv_update_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer,
                              const struct radv_image_view *iview,
                              VkClearDepthStencilValue ds_clear_value, VkImageAspectFlags aspects)
{
   VkImageSubresourceRange range = {
      .aspectMask = iview->vk.aspects,
      .baseMipLevel = iview->vk.base_mip_level,
      .levelCount = iview->vk.level_count,
      .baseArrayLayer = iview->vk.base_array_layer,
      .layerCount = iview->vk.layer_count,
   };
   struct radv_image *image = iview->image;

   assert(radv_htile_enabled(image, range.baseMipLevel));

   radv_set_ds_clear_metadata(cmd_buffer, iview->image, &range, ds_clear_value, aspects);

   if (radv_image_is_tc_compat_htile(image) && (aspects & VK_IMAGE_ASPECT_DEPTH_BIT)) {
      radv_update_tc_compat_zrange_metadata(cmd_buffer, iview, ds_clear_value);
   }

   radv_update_bound_fast_clear_ds(cmd_buffer, iview, ds_clear_value, aspects);
}

/**
 * Load the clear depth/stencil values from the image's metadata.
 */
static void
radv_load_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, const struct radv_image_view *iview)
{
   struct radeon_cmdbuf *cs = cmd_buffer->cs;
   const struct radv_image *image = iview->image;
   VkImageAspectFlags aspects = vk_format_aspects(image->vk.format);
   uint64_t va = radv_get_ds_clear_value_va(image, iview->vk.base_mip_level);
   unsigned reg_offset = 0, reg_count = 0;

   assert(radv_image_has_htile(image));

   if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
      ++reg_count;
   } else {
      ++reg_offset;
      va += 4;
   }
   if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT)
      ++reg_count;

   uint32_t reg = R_028028_DB_STENCIL_CLEAR + 4 * reg_offset;

   if (cmd_buffer->device->physical_device->rad_info.has_load_ctx_reg_pkt) {
      radeon_emit(cs, PKT3(PKT3_LOAD_CONTEXT_REG_INDEX, 3, 0));
      radeon_emit(cs, va);
      radeon_emit(cs, va >> 32);
      radeon_emit(cs, (reg - SI_CONTEXT_REG_OFFSET) >> 2);
      radeon_emit(cs, reg_count);
   } else {
      radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
      radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) | COPY_DATA_DST_SEL(COPY_DATA_REG) |
                         (reg_count == 2 ? COPY_DATA_COUNT_SEL : 0));
      radeon_emit(cs, va);
      radeon_emit(cs, va >> 32);
      radeon_emit(cs, reg >> 2);
      radeon_emit(cs, 0);

      radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0));
      radeon_emit(cs, 0);
   }
}

/*
 * With DCC some colors don't require CMASK elimination before being
 * used as a texture. This sets a predicate value to determine if the
 * cmask eliminate is required.
 */
void
radv_update_fce_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
                         const VkImageSubresourceRange *range, bool value)
{
   if (!image->fce_pred_offset)
      return;

   uint64_t pred_val = value;
   uint64_t va = radv_image_get_fce_pred_va(image, range->baseMipLevel);
   uint32_t level_count = radv_get_levelCount(image, range);
   uint32_t count = 2 * level_count;

   radeon_emit(cmd_buffer->cs, PKT3(PKT3_WRITE_DATA, 2 + count, 0));
   radeon_emit(cmd_buffer->cs,
               S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_PFP));
   radeon_emit(cmd_buffer->cs, va);
   radeon_emit(cmd_buffer->cs, va >> 32);

   for (uint32_t l = 0; l < level_count; l++) {
      radeon_emit(cmd_buffer->cs, pred_val);
      radeon_emit(cmd_buffer->cs, pred_val >> 32);
   }
}

/**
 * Update the DCC predicate to reflect the compression state.
 */
void
radv_update_dcc_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
                         const VkImageSubresourceRange *range, bool value)
{
   if (image->dcc_pred_offset == 0)
      return;

   uint64_t pred_val = value;
   uint64_t va = radv_image_get_dcc_pred_va(image, range->baseMipLevel);
   uint32_t level_count = radv_get_levelCount(image, range);
   uint32_t count = 2 * level_count;

   assert(radv_dcc_enabled(image, range->baseMipLevel));

   radeon_emit(cmd_buffer->cs, PKT3(PKT3_WRITE_DATA, 2 + count, 0));
   radeon_emit(cmd_buffer->cs,
               S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_PFP));
   radeon_emit(cmd_buffer->cs, va);
   radeon_emit(cmd_buffer->cs, va >> 32);

   for (uint32_t l = 0; l < level_count; l++) {
      radeon_emit(cmd_buffer->cs, pred_val);
      radeon_emit(cmd_buffer->cs, pred_val >> 32);
   }
}

/**
 * Update the fast clear color values if the image is bound as a color buffer.
 */
static void
radv_update_bound_fast_clear_color(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
                                   int cb_idx, uint32_t color_values[2])
{
   struct radeon_cmdbuf *cs = cmd_buffer->cs;

   if (cb_idx >= cmd_buffer->state.render.color_att_count ||
       cmd_buffer->state.render.color_att[cb_idx].iview == NULL ||
       cmd_buffer->state.render.color_att[cb_idx].iview->image != image)
      return;

   radeon_set_context_reg_seq(cs, R_028C8C_CB_COLOR0_CLEAR_WORD0 + cb_idx * 0x3c, 2);
   radeon_emit(cs, color_values[0]);
   radeon_emit(cs, color_values[1]);

   cmd_buffer->state.context_roll_without_scissor_emitted = true;
}

/**
 * Set the clear color values to the image's metadata.
 */
static void
radv_set_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
                              const VkImageSubresourceRange *range, uint32_t color_values[2])
{
   struct radeon_cmdbuf *cs = cmd_buffer->cs;
   uint32_t level_count = radv_get_levelCount(image, range);
   uint32_t count = 2 * level_count;

   assert(radv_image_has_cmask(image) || radv_dcc_enabled(image, range->baseMipLevel));

   if (radv_image_has_clear_value(image)) {
      uint64_t va = radv_image_get_fast_clear_va(image, range->baseMipLevel);

      radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + count, cmd_buffer->state.predicating));
      radeon_emit(cs, S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_PFP));
      radeon_emit(cs, va);
      radeon_emit(cs, va >> 32);

      for (uint32_t l = 0; l < level_count; l++) {
         radeon_emit(cs, color_values[0]);
         radeon_emit(cs, color_values[1]);
      }
   } else {
      /* Some default value we can set in the update. */
      assert(color_values[0] == 0 && color_values[1] == 0);
   }
}

/**
 * Update the clear color values for this image.
 */
void
radv_update_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer,
                                 const struct radv_image_view *iview, int cb_idx,
                                 uint32_t color_values[2])
{
   struct radv_image *image = iview->image;
   VkImageSubresourceRange range = {
      .aspectMask = iview->vk.aspects,
      .baseMipLevel = iview->vk.base_mip_level,
      .levelCount = iview->vk.level_count,
      .baseArrayLayer = iview->vk.base_array_layer,
      .layerCount = iview->vk.layer_count,
   };

   assert(radv_image_has_cmask(image) || radv_dcc_enabled(image, iview->vk.base_mip_level));

   /* Do not need to update the clear value for images that are fast cleared with the comp-to-single
    * mode because the hardware gets the value from the image directly.