Google Git
Sign in
fuchsia / third_party / mesa / 2339c4d1cf3b3ea3669ba635392377211af675a8 / . / src / amd / vulkan / radv_pipeline.c
blob: c7f42528999c484d223560bb2cbb8196a6466802 [file] [log] [blame]
/*
* Copyright © 2016 Red Hat.
* Copyright © 2016 Bas Nieuwenhuizen
*
* based in part on anv driver which is:
* Copyright © 2015 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include "nir/nir.h"
#include "nir/nir_builder.h"
#include "spirv/nir_spirv.h"
#include "util/disk_cache.h"
#include "util/mesa-sha1.h"
#include "util/os_time.h"
#include "util/u_atomic.h"
#include "radv_cs.h"
#include "radv_debug.h"
#include "radv_meta.h"
#include "radv_private.h"
#include "radv_shader.h"
#include "radv_shader_args.h"
#include "vk_pipeline.h"
#include "vk_render_pass.h"
#include "vk_util.h"
#include "util/u_debug.h"
#include "ac_binary.h"
#include "ac_nir.h"
#include "ac_shader_util.h"
#include "aco_interface.h"
#include "sid.h"
#include "vk_format.h"
struct radv_blend_state {
uint32_t blend_enable_4bit;
uint32_t need_src_alpha;
uint32_t cb_target_mask;
uint32_t cb_target_enabled_4bit;
uint32_t sx_mrt_blend_opt[8];
uint32_t cb_blend_control[8];
uint32_t spi_shader_col_format;
uint32_t col_format_is_int8;
uint32_t col_format_is_int10;
uint32_t col_format_is_float32;
uint32_t cb_shader_mask;
uint32_t commutative_4bit;
bool mrt0_is_dual_src;
};
struct radv_depth_stencil_state {
uint32_t db_render_control;
uint32_t db_render_override2;
uint32_t db_shader_control;
};
struct radv_dsa_order_invariance {
/* Whether the final result in Z/S buffers is guaranteed to be
* invariant under changes to the order in which fragments arrive.
*/
bool zs;
/* Whether the set of fragments that pass the combined Z/S test is
* guaranteed to be invariant under changes to the order in which
* fragments arrive.
*/
bool pass_set;
};
static bool
radv_is_raster_enabled(const struct radv_graphics_pipeline *pipeline,
const VkGraphicsPipelineCreateInfo *pCreateInfo)
{
return !pCreateInfo->pRasterizationState->rasterizerDiscardEnable ||
(pipeline->dynamic_states & RADV_DYNAMIC_RASTERIZER_DISCARD_ENABLE);
}
static bool
radv_is_static_vrs_enabled(const struct radv_graphics_pipeline *pipeline,
const struct vk_graphics_pipeline_state *state)
{
if (!state->fsr)
return false;
return state->fsr->fragment_size.width != 1 || state->fsr->fragment_size.height != 1 ||
state->fsr->combiner_ops[0] != VK_FRAGMENT_SHADING_RATE_COMBINER_OP_KEEP_KHR ||
state->fsr->combiner_ops[1] != VK_FRAGMENT_SHADING_RATE_COMBINER_OP_KEEP_KHR;
}
static bool
radv_is_vrs_enabled(const struct radv_graphics_pipeline *pipeline,
const struct vk_graphics_pipeline_state *state)
{
return radv_is_static_vrs_enabled(pipeline, state) ||
(pipeline->dynamic_states & RADV_DYNAMIC_FRAGMENT_SHADING_RATE);
}
static bool
radv_pipeline_has_ds_attachments(const struct vk_render_pass_state *rp)
{
return rp->depth_attachment_format != VK_FORMAT_UNDEFINED ||
rp->stencil_attachment_format != VK_FORMAT_UNDEFINED;
}
static bool
radv_pipeline_has_color_attachments(const struct vk_render_pass_state *rp)
{
for (uint32_t i = 0; i < rp->color_attachment_count; ++i) {
if (rp->color_attachment_formats[i] != VK_FORMAT_UNDEFINED)
return true;
}
return false;
}
static bool
radv_pipeline_has_ngg(const struct radv_graphics_pipeline *pipeline)
{
struct radv_shader *shader = pipeline->base.shaders[pipeline->last_vgt_api_stage];
return shader->info.is_ngg;
}
bool
radv_pipeline_has_ngg_passthrough(const struct radv_graphics_pipeline *pipeline)
{
assert(radv_pipeline_has_ngg(pipeline));
struct radv_shader *shader = pipeline->base.shaders[pipeline->last_vgt_api_stage];
return shader->info.is_ngg_passthrough;
}
bool
radv_pipeline_has_gs_copy_shader(const struct radv_pipeline *pipeline)
{
return !!pipeline->gs_copy_shader;
}
static struct radv_pipeline_slab *
radv_pipeline_slab_create(struct radv_device *device, struct radv_pipeline *pipeline,
uint32_t code_size)
{
struct radv_pipeline_slab *slab;
slab = calloc(1, sizeof(*slab));
if (!slab)
return NULL;
slab->ref_count = 1;
slab->alloc = radv_alloc_shader_memory(device, code_size, pipeline);
if (!slab->alloc) {
free(slab);
return NULL;
}
return slab;
}
void
radv_pipeline_slab_destroy(struct radv_device *device, struct radv_pipeline_slab *slab)
{
if (!p_atomic_dec_zero(&slab->ref_count))
return;
radv_free_shader_memory(device, slab->alloc);
free(slab);
}
void
radv_pipeline_destroy(struct radv_device *device, struct radv_pipeline *pipeline,
const VkAllocationCallbacks *allocator)
{
if (pipeline->type == RADV_PIPELINE_GRAPHICS) {
struct radv_graphics_pipeline *graphics_pipeline = radv_pipeline_to_graphics(pipeline);
if (graphics_pipeline->ps_epilog)
radv_shader_part_unref(device, graphics_pipeline->ps_epilog);
vk_free(&device->vk.alloc, graphics_pipeline->state_data);
} else if (pipeline->type == RADV_PIPELINE_RAY_TRACING) {
struct radv_ray_tracing_pipeline *rt_pipeline = radv_pipeline_to_ray_tracing(pipeline);
free(rt_pipeline->group_handles);
free(rt_pipeline->stack_sizes);
} else if (pipeline->type == RADV_PIPELINE_LIBRARY) {
struct radv_library_pipeline *library_pipeline = radv_pipeline_to_library(pipeline);
free(library_pipeline->groups);
for (uint32_t i = 0; i < library_pipeline->stage_count; i++) {
RADV_FROM_HANDLE(vk_shader_module, module, library_pipeline->stages[i].module);
if (module) {
vk_object_base_finish(&module->base);
ralloc_free(module);
}
}
free(library_pipeline->stages);
free(library_pipeline->identifiers);
free(library_pipeline->hashes);
} else if (pipeline->type == RADV_PIPELINE_GRAPHICS_LIB) {
struct radv_graphics_lib_pipeline *gfx_pipeline_lib =
radv_pipeline_to_graphics_lib(pipeline);
radv_pipeline_layout_finish(device, &gfx_pipeline_lib->layout);
for (unsigned i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) {
ralloc_free(pipeline->retained_shaders[i].nir);
}
if (gfx_pipeline_lib->base.ps_epilog)
radv_shader_part_unref(device, gfx_pipeline_lib->base.ps_epilog);
vk_free(&device->vk.alloc, gfx_pipeline_lib->base.state_data);
}
if (pipeline->slab)
radv_pipeline_slab_destroy(device, pipeline->slab);
for (unsigned i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i)
if (pipeline->shaders[i])
radv_shader_unref(device, pipeline->shaders[i]);
if (pipeline->gs_copy_shader)
radv_shader_unref(device, pipeline->gs_copy_shader);
if (pipeline->cs.buf)
free(pipeline->cs.buf);
vk_object_base_finish(&pipeline->base);
vk_free2(&device->vk.alloc, allocator, pipeline);
}
VKAPI_ATTR void VKAPI_CALL
radv_DestroyPipeline(VkDevice _device, VkPipeline _pipeline,
const VkAllocationCallbacks *pAllocator)
{
RADV_FROM_HANDLE(radv_device, device, _device);
RADV_FROM_HANDLE(radv_pipeline, pipeline, _pipeline);
if (!_pipeline)
return;
radv_pipeline_destroy(device, pipeline, pAllocator);
}
uint32_t
radv_get_hash_flags(const struct radv_device *device, bool stats)
{
uint32_t hash_flags = 0;
if (device->physical_device->use_ngg_culling)
hash_flags |= RADV_HASH_SHADER_USE_NGG_CULLING;
if (device->instance->perftest_flags & RADV_PERFTEST_EMULATE_RT)
hash_flags |= RADV_HASH_SHADER_EMULATE_RT;
if (device->physical_device->rt_wave_size == 64)
hash_flags |= RADV_HASH_SHADER_RT_WAVE64;
if (device->physical_device->cs_wave_size == 32)
hash_flags |= RADV_HASH_SHADER_CS_WAVE32;
if (device->physical_device->ps_wave_size == 32)
hash_flags |= RADV_HASH_SHADER_PS_WAVE32;
if (device->physical_device->ge_wave_size == 32)
hash_flags |= RADV_HASH_SHADER_GE_WAVE32;
if (device->physical_device->use_llvm)
hash_flags |= RADV_HASH_SHADER_LLVM;
if (stats)
hash_flags |= RADV_HASH_SHADER_KEEP_STATISTICS;
if (device->robust_buffer_access) /* forces per-attribute vertex descriptors */
hash_flags |= RADV_HASH_SHADER_ROBUST_BUFFER_ACCESS;
if (device->robust_buffer_access2) /* affects load/store vectorizer */
hash_flags |= RADV_HASH_SHADER_ROBUST_BUFFER_ACCESS2;
if (device->instance->debug_flags & RADV_DEBUG_SPLIT_FMA)
hash_flags |= RADV_HASH_SHADER_SPLIT_FMA;
if (device->instance->debug_flags & RADV_DEBUG_NO_FMASK)
hash_flags |= RADV_HASH_SHADER_NO_FMASK;
return hash_flags;
}
static void
radv_pipeline_init_scratch(const struct radv_device *device, struct radv_pipeline *pipeline)
{
unsigned scratch_bytes_per_wave = 0;
unsigned max_waves = 0;
for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) {
if (pipeline->shaders[i] && pipeline->shaders[i]->config.scratch_bytes_per_wave) {
unsigned max_stage_waves = device->scratch_waves;
scratch_bytes_per_wave =
MAX2(scratch_bytes_per_wave, pipeline->shaders[i]->config.scratch_bytes_per_wave);
max_stage_waves =
MIN2(max_stage_waves, 4 * device->physical_device->rad_info.num_cu *
radv_get_max_waves(device, pipeline->shaders[i], i));
max_waves = MAX2(max_waves, max_stage_waves);
}
}
pipeline->scratch_bytes_per_wave = scratch_bytes_per_wave;
pipeline->max_waves = max_waves;
}
static uint32_t
si_translate_blend_function(VkBlendOp op)
{
switch (op) {
case VK_BLEND_OP_ADD:
return V_028780_COMB_DST_PLUS_SRC;
case VK_BLEND_OP_SUBTRACT:
return V_028780_COMB_SRC_MINUS_DST;
case VK_BLEND_OP_REVERSE_SUBTRACT:
return V_028780_COMB_DST_MINUS_SRC;
case VK_BLEND_OP_MIN:
return V_028780_COMB_MIN_DST_SRC;
case VK_BLEND_OP_MAX:
return V_028780_COMB_MAX_DST_SRC;
default:
return 0;
}
}
static uint32_t
si_translate_blend_factor(enum amd_gfx_level gfx_level, VkBlendFactor factor)
{
switch (factor) {
case VK_BLEND_FACTOR_ZERO:
return V_028780_BLEND_ZERO;
case VK_BLEND_FACTOR_ONE:
return V_028780_BLEND_ONE;
case VK_BLEND_FACTOR_SRC_COLOR:
return V_028780_BLEND_SRC_COLOR;
case VK_BLEND_FACTOR_ONE_MINUS_SRC_COLOR:
return V_028780_BLEND_ONE_MINUS_SRC_COLOR;
case VK_BLEND_FACTOR_DST_COLOR:
return V_028780_BLEND_DST_COLOR;
case VK_BLEND_FACTOR_ONE_MINUS_DST_COLOR:
return V_028780_BLEND_ONE_MINUS_DST_COLOR;
case VK_BLEND_FACTOR_SRC_ALPHA:
return V_028780_BLEND_SRC_ALPHA;
case VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA:
return V_028780_BLEND_ONE_MINUS_SRC_ALPHA;
case VK_BLEND_FACTOR_DST_ALPHA:
return V_028780_BLEND_DST_ALPHA;
case VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA:
return V_028780_BLEND_ONE_MINUS_DST_ALPHA;
case VK_BLEND_FACTOR_CONSTANT_COLOR:
return gfx_level >= GFX11 ? V_028780_BLEND_CONSTANT_COLOR_GFX11
: V_028780_BLEND_CONSTANT_COLOR_GFX6;
case VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR:
return gfx_level >= GFX11 ? V_028780_BLEND_ONE_MINUS_CONSTANT_COLOR_GFX11
: V_028780_BLEND_ONE_MINUS_CONSTANT_COLOR_GFX6;
case VK_BLEND_FACTOR_CONSTANT_ALPHA:
return gfx_level >= GFX11 ? V_028780_BLEND_CONSTANT_ALPHA_GFX11
: V_028780_BLEND_CONSTANT_ALPHA_GFX6;
case VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA:
return gfx_level >= GFX11 ? V_028780_BLEND_ONE_MINUS_CONSTANT_ALPHA_GFX11
: V_028780_BLEND_ONE_MINUS_CONSTANT_ALPHA_GFX6;
case VK_BLEND_FACTOR_SRC_ALPHA_SATURATE:
return V_028780_BLEND_SRC_ALPHA_SATURATE;
case VK_BLEND_FACTOR_SRC1_COLOR:
return gfx_level >= GFX11 ? V_028780_BLEND_SRC1_COLOR_GFX11 : V_028780_BLEND_SRC1_COLOR_GFX6;
case VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR:
return gfx_level >= GFX11 ? V_028780_BLEND_INV_SRC1_COLOR_GFX11
: V_028780_BLEND_INV_SRC1_COLOR_GFX6;
case VK_BLEND_FACTOR_SRC1_ALPHA:
return gfx_level >= GFX11 ? V_028780_BLEND_SRC1_ALPHA_GFX11 : V_028780_BLEND_SRC1_ALPHA_GFX6;
case VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA:
return gfx_level >= GFX11 ? V_028780_BLEND_INV_SRC1_ALPHA_GFX11
: V_028780_BLEND_INV_SRC1_ALPHA_GFX6;
default:
return 0;
}
}
static uint32_t
si_translate_blend_opt_function(VkBlendOp op)
{
switch (op) {
case VK_BLEND_OP_ADD:
return V_028760_OPT_COMB_ADD;
case VK_BLEND_OP_SUBTRACT:
return V_028760_OPT_COMB_SUBTRACT;
case VK_BLEND_OP_REVERSE_SUBTRACT:
return V_028760_OPT_COMB_REVSUBTRACT;
case VK_BLEND_OP_MIN:
return V_028760_OPT_COMB_MIN;
case VK_BLEND_OP_MAX:
return V_028760_OPT_COMB_MAX;
default:
return V_028760_OPT_COMB_BLEND_DISABLED;
}
}
static uint32_t
si_translate_blend_opt_factor(VkBlendFactor factor, bool is_alpha)
{
switch (factor) {
case VK_BLEND_FACTOR_ZERO:
return V_028760_BLEND_OPT_PRESERVE_NONE_IGNORE_ALL;
case VK_BLEND_FACTOR_ONE:
return V_028760_BLEND_OPT_PRESERVE_ALL_IGNORE_NONE;
case VK_BLEND_FACTOR_SRC_COLOR:
return is_alpha ? V_028760_BLEND_OPT_PRESERVE_A1_IGNORE_A0
: V_028760_BLEND_OPT_PRESERVE_C1_IGNORE_C0;
case VK_BLEND_FACTOR_ONE_MINUS_SRC_COLOR:
return is_alpha ? V_028760_BLEND_OPT_PRESERVE_A0_IGNORE_A1
: V_028760_BLEND_OPT_PRESERVE_C0_IGNORE_C1;
case VK_BLEND_FACTOR_SRC_ALPHA:
return V_028760_BLEND_OPT_PRESERVE_A1_IGNORE_A0;
case VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA:
return V_028760_BLEND_OPT_PRESERVE_A0_IGNORE_A1;
case VK_BLEND_FACTOR_SRC_ALPHA_SATURATE:
return is_alpha ? V_028760_BLEND_OPT_PRESERVE_ALL_IGNORE_NONE
: V_028760_BLEND_OPT_PRESERVE_NONE_IGNORE_A0;
default:
return V_028760_BLEND_OPT_PRESERVE_NONE_IGNORE_NONE;
}
}
/**
* Get rid of DST in the blend factors by commuting the operands:
* func(src * DST, dst * 0) ---> func(src * 0, dst * SRC)
*/
static void
si_blend_remove_dst(VkBlendOp *func, VkBlendFactor *src_factor, VkBlendFactor *dst_factor,
VkBlendFactor expected_dst, VkBlendFactor replacement_src)
{
if (*src_factor == expected_dst && *dst_factor == VK_BLEND_FACTOR_ZERO) {
*src_factor = VK_BLEND_FACTOR_ZERO;
*dst_factor = replacement_src;
/* Commuting the operands requires reversing subtractions. */
if (*func == VK_BLEND_OP_SUBTRACT)
*func = VK_BLEND_OP_REVERSE_SUBTRACT;
else if (*func == VK_BLEND_OP_REVERSE_SUBTRACT)
*func = VK_BLEND_OP_SUBTRACT;
}
}
static bool
si_blend_factor_uses_dst(VkBlendFactor factor)
{
return factor == VK_BLEND_FACTOR_DST_COLOR || factor == VK_BLEND_FACTOR_DST_ALPHA ||
factor == VK_BLEND_FACTOR_SRC_ALPHA_SATURATE ||
factor == VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA ||
factor == VK_BLEND_FACTOR_ONE_MINUS_DST_COLOR;
}
static bool
is_dual_src(VkBlendFactor factor)
{
switch (factor) {
case VK_BLEND_FACTOR_SRC1_COLOR:
case VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR:
case VK_BLEND_FACTOR_SRC1_ALPHA:
case VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA:
return true;
default:
return false;
}
}
static unsigned
radv_choose_spi_color_format(const struct radv_device *device, VkFormat vk_format,
bool blend_enable, bool blend_need_alpha)
{
const struct util_format_description *desc = vk_format_description(vk_format);
bool use_rbplus = device->physical_device->rad_info.rbplus_allowed;
struct ac_spi_color_formats formats = {0};
unsigned format, ntype, swap;
format = radv_translate_colorformat(vk_format);
ntype = radv_translate_color_numformat(vk_format, desc,
vk_format_get_first_non_void_channel(vk_format));
swap = radv_translate_colorswap(vk_format, false);
ac_choose_spi_color_formats(format, swap, ntype, false, use_rbplus, &formats);
if (blend_enable && blend_need_alpha)
return formats.blend_alpha;
else if (blend_need_alpha)
return formats.alpha;
else if (blend_enable)
return formats.blend;
else
return formats.normal;
}
static bool
format_is_int8(VkFormat format)
{
const struct util_format_description *desc = vk_format_description(format);
int channel = vk_format_get_first_non_void_channel(format);
return channel >= 0 && desc->channel[channel].pure_integer && desc->channel[channel].size == 8;
}
static bool
format_is_int10(VkFormat format)
{
const struct util_format_description *desc = vk_format_description(format);
if (desc->nr_channels != 4)
return false;
for (unsigned i = 0; i < 4; i++) {
if (desc->channel[i].pure_integer && desc->channel[i].size == 10)
return true;
}
return false;
}
static bool
format_is_float32(VkFormat format)
{
const struct util_format_description *desc = vk_format_description(format);
int channel = vk_format_get_first_non_void_channel(format);
return channel >= 0 &&
desc->channel[channel].type == UTIL_FORMAT_TYPE_FLOAT && desc->channel[channel].size == 32;
}
static unsigned
radv_compact_spi_shader_col_format(const struct radv_shader *ps,
const struct radv_blend_state *blend)
{
unsigned spi_shader_col_format = blend->spi_shader_col_format;
unsigned value = 0, num_mrts = 0;
unsigned i, num_targets;
/* Make sure to clear color attachments without exports because MRT holes are removed during
* compilation for optimal performance.
*/
spi_shader_col_format &= ps->info.ps.colors_written;
/* Compute the number of MRTs. */
num_targets = DIV_ROUND_UP(util_last_bit(spi_shader_col_format), 4);
/* Remove holes in spi_shader_col_format. */
for (i = 0; i < num_targets; i++) {
unsigned spi_format = (spi_shader_col_format >> (i * 4)) & 0xf;
if (spi_format) {
value |= spi_format << (num_mrts * 4);
num_mrts++;
}
}
return value;
}
static void
radv_pipeline_compute_spi_color_formats(const struct radv_graphics_pipeline *pipeline,
struct radv_blend_state *blend,
const struct vk_graphics_pipeline_state *state,
bool has_ps_epilog)
{
unsigned col_format = 0, is_int8 = 0, is_int10 = 0, is_float32 = 0;
for (unsigned i = 0; i < state->rp->color_attachment_count; ++i) {
unsigned cf;
VkFormat fmt = state->rp->color_attachment_formats[i];
if (fmt == VK_FORMAT_UNDEFINED || !(blend->cb_target_mask & (0xfu << (i * 4)))) {
cf = V_028714_SPI_SHADER_ZERO;
} else {
bool blend_enable = blend->blend_enable_4bit & (0xfu << (i * 4));
cf = radv_choose_spi_color_format(pipeline->base.device, fmt, blend_enable,
blend->need_src_alpha & (1 << i));
if (format_is_int8(fmt))
is_int8 |= 1 << i;
if (format_is_int10(fmt))
is_int10 |= 1 << i;
if (format_is_float32(fmt))
is_float32 |= 1 << i;
}
col_format |= cf << (4 * i);
}
if (!(col_format & 0xf) && blend->need_src_alpha & (1 << 0)) {
/* When a subpass doesn't have any color attachments, write the
* alpha channel of MRT0 when alpha coverage is enabled because
* the depth attachment needs it.
*/
col_format |= V_028714_SPI_SHADER_32_AR;
}
if (has_ps_epilog) {
/* Do not compact MRTs when the pipeline uses a PS epilog because we can't detect color
* attachments without exports. Without compaction and if the i-th target format is set, all
* previous target formats must be non-zero to avoid hangs.
*/
unsigned num_targets = (util_last_bit(col_format) + 3) / 4;
for (unsigned i = 0; i < num_targets; i++) {
if (!(col_format & (0xfu << (i * 4)))) {
col_format |= V_028714_SPI_SHADER_32_R << (i * 4);
}
}
}
/* The output for dual source blending should have the same format as
* the first output.
*/
if (blend->mrt0_is_dual_src) {
assert(!(col_format >> 4));
col_format |= (col_format & 0xf) << 4;
}
blend->cb_shader_mask = ac_get_cb_shader_mask(col_format);
blend->spi_shader_col_format = col_format;
blend->col_format_is_int8 = is_int8;
blend->col_format_is_int10 = is_int10;
blend->col_format_is_float32 = is_float32;
}
/*
* Ordered so that for each i,
* radv_format_meta_fs_key(radv_fs_key_format_exemplars[i]) == i.
*/
const VkFormat radv_fs_key_format_exemplars[NUM_META_FS_KEYS] = {
VK_FORMAT_R32_SFLOAT,
VK_FORMAT_R32G32_SFLOAT,
VK_FORMAT_R8G8B8A8_UNORM,
VK_FORMAT_R16G16B16A16_UNORM,
VK_FORMAT_R16G16B16A16_SNORM,
VK_FORMAT_R16G16B16A16_UINT,
VK_FORMAT_R16G16B16A16_SINT,
VK_FORMAT_R32G32B32A32_SFLOAT,
VK_FORMAT_R8G8B8A8_UINT,
VK_FORMAT_R8G8B8A8_SINT,
VK_FORMAT_A2R10G10B10_UINT_PACK32,
VK_FORMAT_A2R10G10B10_SINT_PACK32,
};
unsigned
radv_format_meta_fs_key(struct radv_device *device, VkFormat format)
{
unsigned col_format = radv_choose_spi_color_format(device, format, false, false);
assert(col_format != V_028714_SPI_SHADER_32_AR);
bool is_int8 = format_is_int8(format);
bool is_int10 = format_is_int10(format);
if (col_format == V_028714_SPI_SHADER_UINT16_ABGR && is_int8)
return 8;
else if (col_format == V_028714_SPI_SHADER_SINT16_ABGR && is_int8)
return 9;
else if (col_format == V_028714_SPI_SHADER_UINT16_ABGR && is_int10)
return 10;
else if (col_format == V_028714_SPI_SHADER_SINT16_ABGR && is_int10)
return 11;
else {
if (col_format >= V_028714_SPI_SHADER_32_AR)
--col_format; /* Skip V_028714_SPI_SHADER_32_AR since there is no such VkFormat */
--col_format; /* Skip V_028714_SPI_SHADER_ZERO */
return col_format;
}
}
static void
radv_blend_check_commutativity(struct radv_blend_state *blend, VkBlendOp op, VkBlendFactor src,
VkBlendFactor dst, unsigned chanmask)
{
/* Src factor is allowed when it does not depend on Dst. */
static const uint32_t src_allowed =
(1u << VK_BLEND_FACTOR_ONE) | (1u << VK_BLEND_FACTOR_SRC_COLOR) |
(1u << VK_BLEND_FACTOR_SRC_ALPHA) | (1u << VK_BLEND_FACTOR_SRC_ALPHA_SATURATE) |
(1u << VK_BLEND_FACTOR_CONSTANT_COLOR) | (1u << VK_BLEND_FACTOR_CONSTANT_ALPHA) |
(1u << VK_BLEND_FACTOR_SRC1_COLOR) | (1u << VK_BLEND_FACTOR_SRC1_ALPHA) |
(1u << VK_BLEND_FACTOR_ZERO) | (1u << VK_BLEND_FACTOR_ONE_MINUS_SRC_COLOR) |
(1u << VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA) |
(1u << VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR) |
(1u << VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA) |
(1u << VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR) | (1u << VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA);
if (dst == VK_BLEND_FACTOR_ONE && (src_allowed & (1u << src))) {
/* Addition is commutative, but floating point addition isn't
* associative: subtle changes can be introduced via different
* rounding. Be conservative, only enable for min and max.
*/
if (op == VK_BLEND_OP_MAX || op == VK_BLEND_OP_MIN)
blend->commutative_4bit |= chanmask;
}
}
static bool
radv_can_enable_dual_src(const struct vk_color_blend_attachment_state *att)
{
VkBlendOp eqRGB = att->color_blend_op;
VkBlendFactor srcRGB = att->src_color_blend_factor;
VkBlendFactor dstRGB = att->dst_color_blend_factor;
VkBlendOp eqA = att->alpha_blend_op;
VkBlendFactor srcA = att->src_alpha_blend_factor;
VkBlendFactor dstA = att->dst_alpha_blend_factor;
bool eqRGB_minmax = eqRGB == VK_BLEND_OP_MIN || eqRGB == VK_BLEND_OP_MAX;
bool eqA_minmax = eqA == VK_BLEND_OP_MIN || eqA == VK_BLEND_OP_MAX;
assert(att->blend_enable);
if (!eqRGB_minmax && (is_dual_src(srcRGB) || is_dual_src(dstRGB)))
return true;
if (!eqA_minmax && (is_dual_src(srcA) || is_dual_src(dstA)))
return true;
return false;
}
static struct radv_blend_state
radv_pipeline_init_blend_state(struct radv_graphics_pipeline *pipeline,
const struct vk_graphics_pipeline_state *state,
bool has_ps_epilog)
{
const struct radv_device *device = pipeline->base.device;
struct radv_blend_state blend = {0};
unsigned cb_color_control = 0;
const enum amd_gfx_level gfx_level = device->physical_device->rad_info.gfx_level;
int i;
if (state->ms && ((pipeline->dynamic_states & RADV_DYNAMIC_ALPHA_TO_COVERAGE_ENABLE) ||
state->ms->alpha_to_coverage_enable)) {
/* When alpha to coverage is enabled, the driver needs to select a color export format with
* alpha. When this state is dynamic, always select a format with alpha because it's hard to
* change color export formats dynamically (note that it's suboptimal).
*/
blend.need_src_alpha |= 0x1;
}
blend.cb_target_mask = 0;
if (state->cb) {
for (i = 0; i < state->cb->attachment_count; i++) {
unsigned blend_cntl = 0;
unsigned srcRGB_opt, dstRGB_opt, srcA_opt, dstA_opt;
VkBlendOp eqRGB = state->cb->attachments[i].color_blend_op;
VkBlendFactor srcRGB = state->cb->attachments[i].src_color_blend_factor;
VkBlendFactor dstRGB = state->cb->attachments[i].dst_color_blend_factor;
VkBlendOp eqA = state->cb->attachments[i].alpha_blend_op;
VkBlendFactor srcA = state->cb->attachments[i].src_alpha_blend_factor;
VkBlendFactor dstA = state->cb->attachments[i].dst_alpha_blend_factor;
blend.sx_mrt_blend_opt[i] = S_028760_COLOR_COMB_FCN(V_028760_OPT_COMB_BLEND_DISABLED) |
S_028760_ALPHA_COMB_FCN(V_028760_OPT_COMB_BLEND_DISABLED);
if (!state->cb->attachments[i].write_mask)
continue;
/* Ignore other blend targets if dual-source blending
* is enabled to prevent wrong behaviour.
*/
if (blend.mrt0_is_dual_src)
continue;
blend.cb_target_mask |= (unsigned)state->cb->attachments[i].write_mask << (4 * i);
blend.cb_target_enabled_4bit |= 0xfu << (4 * i);
if (!state->cb->attachments[i].blend_enable) {
blend.cb_blend_control[i] = blend_cntl;
continue;
}
if (i == 0 && radv_can_enable_dual_src(&state->cb->attachments[i])) {
blend.mrt0_is_dual_src = true;
}
if (eqRGB == VK_BLEND_OP_MIN || eqRGB == VK_BLEND_OP_MAX) {
srcRGB = VK_BLEND_FACTOR_ONE;
dstRGB = VK_BLEND_FACTOR_ONE;
}
if (eqA == VK_BLEND_OP_MIN || eqA == VK_BLEND_OP_MAX) {
srcA = VK_BLEND_FACTOR_ONE;
dstA = VK_BLEND_FACTOR_ONE;
}
radv_blend_check_commutativity(&blend, eqRGB, srcRGB, dstRGB, 0x7u << (4 * i));
radv_blend_check_commutativity(&blend, eqA, srcA, dstA, 0x8u << (4 * i));
/* Blending optimizations for RB+.
* These transformations don't change the behavior.
*
* First, get rid of DST in the blend factors:
* func(src * DST, dst * 0) ---> func(src * 0, dst * SRC)
*/
si_blend_remove_dst(&eqRGB, &srcRGB, &dstRGB, VK_BLEND_FACTOR_DST_COLOR,
VK_BLEND_FACTOR_SRC_COLOR);
si_blend_remove_dst(&eqA, &srcA, &dstA, VK_BLEND_FACTOR_DST_COLOR,
VK_BLEND_FACTOR_SRC_COLOR);
si_blend_remove_dst(&eqA, &srcA, &dstA, VK_BLEND_FACTOR_DST_ALPHA,
VK_BLEND_FACTOR_SRC_ALPHA);
/* Look up the ideal settings from tables. */
srcRGB_opt = si_translate_blend_opt_factor(srcRGB, false);
dstRGB_opt = si_translate_blend_opt_factor(dstRGB, false);
srcA_opt = si_translate_blend_opt_factor(srcA, true);
dstA_opt = si_translate_blend_opt_factor(dstA, true);
/* Handle interdependencies. */
if (si_blend_factor_uses_dst(srcRGB))
dstRGB_opt = V_028760_BLEND_OPT_PRESERVE_NONE_IGNORE_NONE;
if (si_blend_factor_uses_dst(srcA))
dstA_opt = V_028760_BLEND_OPT_PRESERVE_NONE_IGNORE_NONE;
if (srcRGB == VK_BLEND_FACTOR_SRC_ALPHA_SATURATE &&
(dstRGB == VK_BLEND_FACTOR_ZERO || dstRGB == VK_BLEND_FACTOR_SRC_ALPHA ||
dstRGB == VK_BLEND_FACTOR_SRC_ALPHA_SATURATE))
dstRGB_opt = V_028760_BLEND_OPT_PRESERVE_NONE_IGNORE_A0;
/* Set the final value. */
blend.sx_mrt_blend_opt[i] =
S_028760_COLOR_SRC_OPT(srcRGB_opt) | S_028760_COLOR_DST_OPT(dstRGB_opt) |
S_028760_COLOR_COMB_FCN(si_translate_blend_opt_function(eqRGB)) |
S_028760_ALPHA_SRC_OPT(srcA_opt) | S_028760_ALPHA_DST_OPT(dstA_opt) |
S_028760_ALPHA_COMB_FCN(si_translate_blend_opt_function(eqA));
blend_cntl |= S_028780_ENABLE(1);
blend_cntl |= S_028780_COLOR_COMB_FCN(si_translate_blend_function(eqRGB));
blend_cntl |= S_028780_COLOR_SRCBLEND(si_translate_blend_factor(gfx_level, srcRGB));
blend_cntl |= S_028780_COLOR_DESTBLEND(si_translate_blend_factor(gfx_level, dstRGB));
if (srcA != srcRGB || dstA != dstRGB || eqA != eqRGB) {
blend_cntl |= S_028780_SEPARATE_ALPHA_BLEND(1);
blend_cntl |= S_028780_ALPHA_COMB_FCN(si_translate_blend_function(eqA));
blend_cntl |= S_028780_ALPHA_SRCBLEND(si_translate_blend_factor(gfx_level, srcA));
blend_cntl |= S_028780_ALPHA_DESTBLEND(si_translate_blend_factor(gfx_level, dstA));
}
blend.cb_blend_control[i] = blend_cntl;
blend.blend_enable_4bit |= 0xfu << (i * 4);
if (srcRGB == VK_BLEND_FACTOR_SRC_ALPHA || dstRGB == VK_BLEND_FACTOR_SRC_ALPHA ||
srcRGB == VK_BLEND_FACTOR_SRC_ALPHA_SATURATE ||
dstRGB == VK_BLEND_FACTOR_SRC_ALPHA_SATURATE ||
srcRGB == VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA ||
dstRGB == VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA)
blend.need_src_alpha |= 1 << i;
}
for (i = state->cb->attachment_count; i < 8; i++) {
blend.cb_blend_control[i] = 0;
blend.sx_mrt_blend_opt[i] = S_028760_COLOR_COMB_FCN(V_028760_OPT_COMB_BLEND_DISABLED) |
S_028760_ALPHA_COMB_FCN(V_028760_OPT_COMB_BLEND_DISABLED);
}
}
if (device->physical_device->rad_info.has_rbplus) {
/* Disable RB+ blend optimizations for dual source blending. */
if (blend.mrt0_is_dual_src) {
for (i = 0; i < 8; i++) {
blend.sx_mrt_blend_opt[i] = S_028760_COLOR_COMB_FCN(V_028760_OPT_COMB_NONE) |
S_028760_ALPHA_COMB_FCN(V_028760_OPT_COMB_NONE);
}
}
/* RB+ doesn't work with dual source blending, logic op and
* RESOLVE.
*/
if (blend.mrt0_is_dual_src || (state->cb && state->cb->logic_op_enable) ||
(device->physical_device->rad_info.gfx_level >= GFX11 && blend.blend_enable_4bit))
cb_color_control |= S_028808_DISABLE_DUAL_QUAD(1);
}
if (blend.cb_target_mask)
cb_color_control |= S_028808_MODE(V_028808_CB_NORMAL);
else
cb_color_control |= S_028808_MODE(V_028808_CB_DISABLE);
if (state->rp)
radv_pipeline_compute_spi_color_formats(pipeline, &blend, state, has_ps_epilog);
pipeline->cb_color_control = cb_color_control;
return blend;
}
static unsigned
radv_pipeline_color_samples(const struct vk_graphics_pipeline_state *state)
{
if (radv_pipeline_has_color_attachments(state->rp)) {
unsigned color_attachment_samples = 0;
for (uint32_t i = 0; i < state->rp->color_attachment_count; i++) {
if (state->rp->color_attachment_formats[i] != VK_FORMAT_UNDEFINED) {
color_attachment_samples =
MAX2(color_attachment_samples, state->rp->color_attachment_samples[i]);
}
}
if (color_attachment_samples)
return color_attachment_samples;
}
return state->ms ? state->ms->rasterization_samples : 1;
}
static unsigned
radv_pipeline_depth_samples(const struct vk_graphics_pipeline_state *state)
{
if (state->rp->depth_stencil_attachment_samples && radv_pipeline_has_ds_attachments(state->rp)) {
return state->rp->depth_stencil_attachment_samples;
}
return state->ms ? state->ms->rasterization_samples : 1;
}
static uint8_t
radv_pipeline_get_ps_iter_samples(const struct vk_graphics_pipeline_state *state)
{
uint32_t ps_iter_samples = 1;
uint32_t num_samples = radv_pipeline_color_samples(state);
if (state->ms && state->ms->sample_shading_enable) {
ps_iter_samples = ceilf(state->ms->min_sample_shading * num_samples);
ps_iter_samples = util_next_power_of_two(ps_iter_samples);
}
return ps_iter_samples;
}
static bool
radv_is_depth_write_enabled(const struct vk_depth_stencil_state *ds)
{
return ds->depth.test_enable && ds->depth.write_enable &&
ds->depth.compare_op != VK_COMPARE_OP_NEVER;
}
static bool
radv_writes_stencil(const struct vk_stencil_test_face_state *face)
{
return face->write_mask &&
(face->op.fail != VK_STENCIL_OP_KEEP || face->op.pass != VK_STENCIL_OP_KEEP ||
face->op.depth_fail != VK_STENCIL_OP_KEEP);
}
static bool
radv_is_stencil_write_enabled(const struct vk_depth_stencil_state *ds)
{
return ds->stencil.test_enable &&
(radv_writes_stencil(&ds->stencil.front) || radv_writes_stencil(&ds->stencil.back));
}
static bool
radv_order_invariant_stencil_op(VkStencilOp op)
{
/* REPLACE is normally order invariant, except when the stencil
* reference value is written by the fragment shader. Tracking this
* interaction does not seem worth the effort, so be conservative.
*/
return op != VK_STENCIL_OP_INCREMENT_AND_CLAMP && op != VK_STENCIL_OP_DECREMENT_AND_CLAMP &&
op != VK_STENCIL_OP_REPLACE;
}
static bool
radv_order_invariant_stencil_state(const struct vk_stencil_test_face_state *face)
{
/* Compute whether, assuming Z writes are disabled, this stencil state
* is order invariant in the sense that the set of passing fragments as
* well as the final stencil buffer result does not depend on the order
* of fragments.
*/
return !face->write_mask ||
/* The following assumes that Z writes are disabled. */
(face->op.compare == VK_COMPARE_OP_ALWAYS &&
radv_order_invariant_stencil_op(face->op.pass) &&
radv_order_invariant_stencil_op(face->op.depth_fail)) ||
(face->op.compare == VK_COMPARE_OP_NEVER &&
radv_order_invariant_stencil_op(face->op.fail));
}
static bool
radv_pipeline_has_dynamic_ds_states(const struct radv_graphics_pipeline *pipeline)
{
return !!(pipeline->dynamic_states & (RADV_DYNAMIC_DEPTH_TEST_ENABLE |
RADV_DYNAMIC_DEPTH_WRITE_ENABLE |
RADV_DYNAMIC_DEPTH_COMPARE_OP |
RADV_DYNAMIC_STENCIL_TEST_ENABLE |
RADV_DYNAMIC_STENCIL_WRITE_MASK |
RADV_DYNAMIC_STENCIL_OP));
}
static bool
radv_pipeline_out_of_order_rast(struct radv_graphics_pipeline *pipeline,
const struct radv_blend_state *blend,
const struct vk_graphics_pipeline_state *state)
{
unsigned colormask = blend->cb_target_enabled_4bit;
if (!pipeline->base.device->physical_device->out_of_order_rast_allowed)
return false;
/* Be conservative if a logic operation is enabled with color buffers. */
if (colormask &&
((pipeline->dynamic_states & RADV_DYNAMIC_LOGIC_OP_ENABLE) || state->cb->logic_op_enable))
return false;
/* Be conservative if an extended dynamic depth/stencil state is
* enabled because the driver can't update out-of-order rasterization
* dynamically.
*/
if (radv_pipeline_has_dynamic_ds_states(pipeline))
return false;
/* Default depth/stencil invariance when no attachment is bound. */
struct radv_dsa_order_invariance dsa_order_invariant = {.zs = true, .pass_set = true};
if (state->ds) {
bool has_stencil = state->rp->stencil_attachment_format != VK_FORMAT_UNDEFINED;
struct radv_dsa_order_invariance order_invariance[2];
struct radv_shader *ps = pipeline->base.shaders[MESA_SHADER_FRAGMENT];
/* Compute depth/stencil order invariance in order to know if
* it's safe to enable out-of-order.
*/
bool zfunc_is_ordered = state->ds->depth.compare_op == VK_COMPARE_OP_NEVER ||
state->ds->depth.compare_op == VK_COMPARE_OP_LESS ||
state->ds->depth.compare_op == VK_COMPARE_OP_LESS_OR_EQUAL ||
state->ds->depth.compare_op == VK_COMPARE_OP_GREATER ||
state->ds->depth.compare_op == VK_COMPARE_OP_GREATER_OR_EQUAL;
bool depth_write_enabled = radv_is_depth_write_enabled(state->ds);
bool stencil_write_enabled = radv_is_stencil_write_enabled(state->ds);
bool ds_write_enabled = depth_write_enabled || stencil_write_enabled;
bool nozwrite_and_order_invariant_stencil =
!ds_write_enabled ||
(!depth_write_enabled && radv_order_invariant_stencil_state(&state->ds->stencil.front) &&
radv_order_invariant_stencil_state(&state->ds->stencil.back));
order_invariance[1].zs = nozwrite_and_order_invariant_stencil ||
(!stencil_write_enabled && zfunc_is_ordered);
order_invariance[0].zs = !depth_write_enabled || zfunc_is_ordered;
order_invariance[1].pass_set =
nozwrite_and_order_invariant_stencil ||
(!stencil_write_enabled &&
(state->ds->depth.compare_op == VK_COMPARE_OP_ALWAYS ||
state->ds->depth.compare_op == VK_COMPARE_OP_NEVER));
order_invariance[0].pass_set =
!depth_write_enabled ||
(state->ds->depth.compare_op == VK_COMPARE_OP_ALWAYS ||
state->ds->depth.compare_op == VK_COMPARE_OP_NEVER);
dsa_order_invariant = order_invariance[has_stencil];
if (!dsa_order_invariant.zs)
return false;
/* The set of PS invocations is always order invariant,
* except when early Z/S tests are requested.
*/
if (ps && ps->info.ps.writes_memory && ps->info.ps.early_fragment_test &&
!dsa_order_invariant.pass_set)
return false;
/* Determine if out-of-order rasterization should be disabled when occlusion queries are used. */
pipeline->disable_out_of_order_rast_for_occlusion = !dsa_order_invariant.pass_set;
}
/* No color buffers are enabled for writing. */
if (!colormask)
return true;
unsigned blendmask = colormask & blend->blend_enable_4bit;
if (blendmask) {
/* Only commutative blending. */
if (blendmask & ~blend->commutative_4bit)
return false;
if (!dsa_order_invariant.pass_set)
return false;
}
if (colormask & ~blendmask)
return false;
return true;
}
static void
radv_pipeline_init_multisample_state(struct radv_graphics_pipeline *pipeline,
const struct radv_blend_state *blend,
const struct vk_graphics_pipeline_state *state,
unsigned rast_prim)
{
const struct radv_physical_device *pdevice = pipeline->base.device->physical_device;
struct radv_multisample_state *ms = &pipeline->ms;
unsigned num_tile_pipes = pdevice->rad_info.num_tile_pipes;
bool out_of_order_rast = false;
int ps_iter_samples = 1;
ms->num_samples = state->ms ? state->ms->rasterization_samples : 1;
/* From the Vulkan 1.1.129 spec, 26.7. Sample Shading:
*
* "Sample shading is enabled for a graphics pipeline:
*
* - If the interface of the fragment shader entry point of the
* graphics pipeline includes an input variable decorated
* with SampleId or SamplePosition. In this case
* minSampleShadingFactor takes the value 1.0.
* - Else if the sampleShadingEnable member of the
* VkPipelineMultisampleStateCreateInfo structure specified
* when creating the graphics pipeline is set to VK_TRUE. In
* this case minSampleShadingFactor takes the value of
* VkPipelineMultisampleStateCreateInfo::minSampleShading.
*
* Otherwise, sample shading is considered disabled."
*/
if (pipeline->base.shaders[MESA_SHADER_FRAGMENT]->info.ps.uses_sample_shading) {
ps_iter_samples = ms->num_samples;
} else {
ps_iter_samples = radv_pipeline_get_ps_iter_samples(state);
}
if (state->rs->rasterization_order_amd == VK_RASTERIZATION_ORDER_RELAXED_AMD) {
/* Out-of-order rasterization is explicitly enabled by the
* application.
*/
out_of_order_rast = true;
} else {
/* Determine if the driver can enable out-of-order
* rasterization internally.
*/
out_of_order_rast = radv_pipeline_out_of_order_rast(pipeline, blend, state);
}
ms->pa_sc_aa_config = 0;
ms->db_eqaa = S_028804_HIGH_QUALITY_INTERSECTIONS(1) | S_028804_INCOHERENT_EQAA_READS(1) |
S_028804_INTERPOLATE_COMP_Z(pdevice->rad_info.gfx_level < GFX11) |
S_028804_STATIC_ANCHOR_ASSOCIATIONS(1);
ms->pa_sc_mode_cntl_1 =
S_028A4C_WALK_FENCE_ENABLE(1) | // TODO linear dst fixes
S_028A4C_WALK_FENCE_SIZE(num_tile_pipes == 2 ? 2 : 3) |
S_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE(out_of_order_rast) |
S_028A4C_OUT_OF_ORDER_WATER_MARK(0x7) |
/* always 1: */
S_028A4C_WALK_ALIGN8_PRIM_FITS_ST(1) | S_028A4C_SUPERTILE_WALK_ORDER_ENABLE(1) |
S_028A4C_TILE_WALK_ORDER_ENABLE(1) | S_028A4C_MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE(1) |
S_028A4C_FORCE_EOV_CNTDWN_ENABLE(1) | S_028A4C_FORCE_EOV_REZ_ENABLE(1);
ms->pa_sc_mode_cntl_0 = S_028A48_ALTERNATE_RBS_PER_TILE(pdevice->rad_info.gfx_level >= GFX9) |
S_028A48_VPORT_SCISSOR_ENABLE(1);
if (state->rs->line.mode == VK_LINE_RASTERIZATION_MODE_BRESENHAM_EXT &&
radv_rast_prim_is_line(rast_prim)) {
/* From the Vulkan spec 1.3.221:
*
* "When Bresenham lines are being rasterized, sample locations may all be treated as being at
* the pixel center (this may affect attribute and depth interpolation)."
*
* "One consequence of this is that Bresenham lines cover the same pixels regardless of the
* number of rasterization samples, and cover all samples in those pixels (unless masked out
* or killed)."
*/
ms->num_samples = 1;
}
if (ms->num_samples > 1) {
uint32_t z_samples = radv_pipeline_depth_samples(state);
unsigned log_samples = util_logbase2(ms->num_samples);
unsigned log_z_samples = util_logbase2(z_samples);
unsigned log_ps_iter_samples = util_logbase2(ps_iter_samples);
ms->pa_sc_mode_cntl_0 |= S_028A48_MSAA_ENABLE(1);
ms->db_eqaa |= S_028804_MAX_ANCHOR_SAMPLES(log_z_samples) |
S_028804_PS_ITER_SAMPLES(log_ps_iter_samples) |
S_028804_MASK_EXPORT_NUM_SAMPLES(log_samples) |
S_028804_ALPHA_TO_MASK_NUM_SAMPLES(log_samples);
ms->pa_sc_aa_config |=
S_028BE0_MSAA_NUM_SAMPLES(log_samples) |
S_028BE0_MAX_SAMPLE_DIST(radv_get_default_max_sample_dist(log_samples)) |
S_028BE0_MSAA_EXPOSED_SAMPLES(log_samples) | /* CM_R_028BE0_PA_SC_AA_CONFIG */
S_028BE0_COVERED_CENTROID_IS_CENTER(pdevice->rad_info.gfx_level >= GFX10_3);
ms->pa_sc_mode_cntl_1 |= S_028A4C_PS_ITER_SAMPLE(ps_iter_samples > 1);
if (ps_iter_samples > 1)
pipeline->spi_baryc_cntl |= S_0286E0_POS_FLOAT_LOCATION(2);
}
}
static void
gfx103_pipeline_init_vrs_state(struct radv_graphics_pipeline *pipeline,
const struct vk_graphics_pipeline_state *state)
{
struct radv_shader *ps = pipeline->base.shaders[MESA_SHADER_FRAGMENT];
struct radv_multisample_state *ms = &pipeline->ms;
struct radv_vrs_state *vrs = &pipeline->vrs;
if ((state->ms && state->ms->sample_shading_enable) ||
ps->info.ps.uses_sample_shading || ps->info.ps.reads_sample_mask_in) {
/* Disable VRS and use the rates from PS_ITER_SAMPLES if:
*
* 1) sample shading is enabled or per-sample interpolation is
* used by the fragment shader
* 2) the fragment shader reads gl_SampleMaskIn because the
* 16-bit sample coverage mask isn't enough for MSAA8x and
* 2x2 coarse shading isn't enough.
*/
vrs->pa_cl_vrs_cntl = S_028848_SAMPLE_ITER_COMBINER_MODE(V_028848_VRS_COMB_MODE_OVERRIDE);
/* Make sure sample shading is enabled even if only MSAA1x is
* used because the SAMPLE_ITER combiner is in passthrough
* mode if PS_ITER_SAMPLE is 0, and it uses the per-draw rate.
* The default VRS rate when sample shading is enabled is 1x1.
*/
if (!G_028A4C_PS_ITER_SAMPLE(ms->pa_sc_mode_cntl_1))
ms->pa_sc_mode_cntl_1 |= S_028A4C_PS_ITER_SAMPLE(1);
} else {
vrs->pa_cl_vrs_cntl = S_028848_SAMPLE_ITER_COMBINER_MODE(V_028848_VRS_COMB_MODE_PASSTHRU);
}
}
static uint32_t
si_conv_tess_prim_to_gs_out(enum tess_primitive_mode prim)
{
switch (prim) {
case TESS_PRIMITIVE_TRIANGLES:
case TESS_PRIMITIVE_QUADS:
return V_028A6C_TRISTRIP;
case TESS_PRIMITIVE_ISOLINES:
return V_028A6C_LINESTRIP;
default:
assert(0);
return 0;
}
}
static uint32_t
si_conv_gl_prim_to_gs_out(unsigned gl_prim)
{
switch (gl_prim) {
case SHADER_PRIM_POINTS:
return V_028A6C_POINTLIST;
case SHADER_PRIM_LINES:
case SHADER_PRIM_LINE_STRIP:
case SHADER_PRIM_LINES_ADJACENCY:
return V_028A6C_LINESTRIP;
case SHADER_PRIM_TRIANGLES:
case SHADER_PRIM_TRIANGLE_STRIP_ADJACENCY:
case SHADER_PRIM_TRIANGLE_STRIP:
case SHADER_PRIM_QUADS:
return V_028A6C_TRISTRIP;
default:
assert(0);
return 0;
}
}
static uint64_t
radv_dynamic_state_mask(VkDynamicState state)
{
switch (state) {
case VK_DYNAMIC_STATE_VIEWPORT:
case VK_DYNAMIC_STATE_VIEWPORT_WITH_COUNT:
return RADV_DYNAMIC_VIEWPORT;
case VK_DYNAMIC_STATE_SCISSOR:
case VK_DYNAMIC_STATE_SCISSOR_WITH_COUNT:
return RADV_DYNAMIC_SCISSOR;
case VK_DYNAMIC_STATE_LINE_WIDTH:
return RADV_DYNAMIC_LINE_WIDTH;
case VK_DYNAMIC_STATE_DEPTH_BIAS:
return RADV_DYNAMIC_DEPTH_BIAS;
case VK_DYNAMIC_STATE_BLEND_CONSTANTS:
return RADV_DYNAMIC_BLEND_CONSTANTS;
case VK_DYNAMIC_STATE_DEPTH_BOUNDS:
return RADV_DYNAMIC_DEPTH_BOUNDS;
case VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK:
return RADV_DYNAMIC_STENCIL_COMPARE_MASK;
case VK_DYNAMIC_STATE_STENCIL_WRITE_MASK:
return RADV_DYNAMIC_STENCIL_WRITE_MASK;
case VK_DYNAMIC_STATE_STENCIL_REFERENCE:
return RADV_DYNAMIC_STENCIL_REFERENCE;
case VK_DYNAMIC_STATE_DISCARD_RECTANGLE_EXT:
return RADV_DYNAMIC_DISCARD_RECTANGLE;
case VK_DYNAMIC_STATE_SAMPLE_LOCATIONS_EXT:
return RADV_DYNAMIC_SAMPLE_LOCATIONS;
case VK_DYNAMIC_STATE_LINE_STIPPLE_EXT:
return RADV_DYNAMIC_LINE_STIPPLE;
case VK_DYNAMIC_STATE_CULL_MODE:
return RADV_DYNAMIC_CULL_MODE;
case VK_DYNAMIC_STATE_FRONT_FACE:
return RADV_DYNAMIC_FRONT_FACE;
case VK_DYNAMIC_STATE_PRIMITIVE_TOPOLOGY:
return RADV_DYNAMIC_PRIMITIVE_TOPOLOGY;
case VK_DYNAMIC_STATE_DEPTH_TEST_ENABLE:
return RADV_DYNAMIC_DEPTH_TEST_ENABLE;
case VK_DYNAMIC_STATE_DEPTH_WRITE_ENABLE:
return RADV_DYNAMIC_DEPTH_WRITE_ENABLE;
case VK_DYNAMIC_STATE_DEPTH_COMPARE_OP:
return RADV_DYNAMIC_DEPTH_COMPARE_OP;
case VK_DYNAMIC_STATE_DEPTH_BOUNDS_TEST_ENABLE:
return RADV_DYNAMIC_DEPTH_BOUNDS_TEST_ENABLE;
case VK_DYNAMIC_STATE_STENCIL_TEST_ENABLE:
return RADV_DYNAMIC_STENCIL_TEST_ENABLE;
case VK_DYNAMIC_STATE_STENCIL_OP:
return RADV_DYNAMIC_STENCIL_OP;
case VK_DYNAMIC_STATE_VERTEX_INPUT_BINDING_STRIDE:
return RADV_DYNAMIC_VERTEX_INPUT_BINDING_STRIDE;
case VK_DYNAMIC_STATE_FRAGMENT_SHADING_RATE_KHR:
return RADV_DYNAMIC_FRAGMENT_SHADING_RATE;
case VK_DYNAMIC_STATE_PATCH_CONTROL_POINTS_EXT:
return RADV_DYNAMIC_PATCH_CONTROL_POINTS;
case VK_DYNAMIC_STATE_RASTERIZER_DISCARD_ENABLE:
return RADV_DYNAMIC_RASTERIZER_DISCARD_ENABLE;
case VK_DYNAMIC_STATE_DEPTH_BIAS_ENABLE:
return RADV_DYNAMIC_DEPTH_BIAS_ENABLE;
case VK_DYNAMIC_STATE_LOGIC_OP_EXT:
return RADV_DYNAMIC_LOGIC_OP;
case VK_DYNAMIC_STATE_PRIMITIVE_RESTART_ENABLE:
return RADV_DYNAMIC_PRIMITIVE_RESTART_ENABLE;
case VK_DYNAMIC_STATE_COLOR_WRITE_ENABLE_EXT:
return RADV_DYNAMIC_COLOR_WRITE_ENABLE;
case VK_DYNAMIC_STATE_VERTEX_INPUT_EXT:
return RADV_DYNAMIC_VERTEX_INPUT;
case VK_DYNAMIC_STATE_POLYGON_MODE_EXT:
return RADV_DYNAMIC_POLYGON_MODE;
case VK_DYNAMIC_STATE_TESSELLATION_DOMAIN_ORIGIN_EXT:
return RADV_DYNAMIC_TESS_DOMAIN_ORIGIN;
case VK_DYNAMIC_STATE_LOGIC_OP_ENABLE_EXT:
return RADV_DYNAMIC_LOGIC_OP_ENABLE;
case VK_DYNAMIC_STATE_LINE_STIPPLE_ENABLE_EXT:
return RADV_DYNAMIC_LINE_STIPPLE_ENABLE;
case VK_DYNAMIC_STATE_ALPHA_TO_COVERAGE_ENABLE_EXT:
return RADV_DYNAMIC_ALPHA_TO_COVERAGE_ENABLE;
case VK_DYNAMIC_STATE_SAMPLE_MASK_EXT:
return RADV_DYNAMIC_SAMPLE_MASK;
case VK_DYNAMIC_STATE_DEPTH_CLIP_ENABLE_EXT:
return RADV_DYNAMIC_DEPTH_CLIP_ENABLE;
case VK_DYNAMIC_STATE_CONSERVATIVE_RASTERIZATION_MODE_EXT:
return RADV_DYNAMIC_CONSERVATIVE_RAST_MODE;
case VK_DYNAMIC_STATE_DEPTH_CLIP_NEGATIVE_ONE_TO_ONE_EXT:
return RADV_DYNAMIC_DEPTH_CLIP_NEGATIVE_ONE_TO_ONE;
case VK_DYNAMIC_STATE_PROVOKING_VERTEX_MODE_EXT:
return RADV_DYNAMIC_PROVOKING_VERTEX_MODE;
case VK_DYNAMIC_STATE_DEPTH_CLAMP_ENABLE_EXT:
return RADV_DYNAMIC_DEPTH_CLAMP_ENABLE;
default:
unreachable("Unhandled dynamic state");
}
}
static bool
radv_pipeline_is_blend_enabled(const struct radv_graphics_pipeline *pipeline,
const struct vk_color_blend_state *cb)
{
if (cb) {
for (uint32_t i = 0; i < cb->attachment_count; i++) {
if (cb->attachments[i].write_mask && cb->attachments[i].blend_enable)
return true;
}
}
return false;
}
static uint64_t
radv_pipeline_needed_dynamic_state(const struct radv_graphics_pipeline *pipeline,
const struct vk_graphics_pipeline_state *state)
{
bool has_color_att = radv_pipeline_has_color_attachments(state->rp);
bool raster_enabled = !state->rs->rasterizer_discard_enable ||
(pipeline->dynamic_states & RADV_DYNAMIC_RASTERIZER_DISCARD_ENABLE);
uint64_t states = RADV_DYNAMIC_ALL;
/* Disable dynamic states that are useless to mesh shading. */
if (radv_pipeline_has_stage(pipeline, MESA_SHADER_MESH)) {
if (!raster_enabled)
return RADV_DYNAMIC_RASTERIZER_DISCARD_ENABLE;
states &= ~(RADV_DYNAMIC_VERTEX_INPUT | RADV_DYNAMIC_VERTEX_INPUT_BINDING_STRIDE |
RADV_DYNAMIC_PRIMITIVE_RESTART_ENABLE | RADV_DYNAMIC_PRIMITIVE_TOPOLOGY);
}
/* Disable dynamic states that are useless when rasterization is disabled. */
if (!raster_enabled) {
states = RADV_DYNAMIC_PRIMITIVE_TOPOLOGY | RADV_DYNAMIC_VERTEX_INPUT_BINDING_STRIDE |
RADV_DYNAMIC_PRIMITIVE_RESTART_ENABLE | RADV_DYNAMIC_RASTERIZER_DISCARD_ENABLE |
RADV_DYNAMIC_VERTEX_INPUT;
if (pipeline->active_stages & VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT)
states |= RADV_DYNAMIC_PATCH_CONTROL_POINTS | RADV_DYNAMIC_TESS_DOMAIN_ORIGIN;
return states;
}
if (!state->rs->depth_bias.enable &&
!(pipeline->dynamic_states & RADV_DYNAMIC_DEPTH_BIAS_ENABLE))
states &= ~RADV_DYNAMIC_DEPTH_BIAS;
if (!(pipeline->dynamic_states & RADV_DYNAMIC_DEPTH_BOUNDS_TEST_ENABLE) &&
(!state->ds || !state->ds->depth.bounds_test.enable))
states &= ~RADV_DYNAMIC_DEPTH_BOUNDS;
if (!(pipeline->dynamic_states & RADV_DYNAMIC_STENCIL_TEST_ENABLE) &&
(!state->ds || !state->ds->stencil.test_enable))
states &= ~(RADV_DYNAMIC_STENCIL_COMPARE_MASK | RADV_DYNAMIC_STENCIL_WRITE_MASK |
RADV_DYNAMIC_STENCIL_REFERENCE | RADV_DYNAMIC_STENCIL_OP);
if (!state->dr->rectangle_count)
states &= ~RADV_DYNAMIC_DISCARD_RECTANGLE;
if (!state->ms || !state->ms->sample_locations_enable)
states &= ~RADV_DYNAMIC_SAMPLE_LOCATIONS;
if (!(pipeline->dynamic_states & RADV_DYNAMIC_LINE_STIPPLE_ENABLE) &&
!state->rs->line.stipple.enable)
states &= ~RADV_DYNAMIC_LINE_STIPPLE;
if (!radv_is_vrs_enabled(pipeline, state))
states &= ~RADV_DYNAMIC_FRAGMENT_SHADING_RATE;
if (!has_color_att || !radv_pipeline_is_blend_enabled(pipeline, state->cb))
states &= ~RADV_DYNAMIC_BLEND_CONSTANTS;
if (!has_color_att)
states &= ~RADV_DYNAMIC_COLOR_WRITE_ENABLE;
if (!(pipeline->active_stages & VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT))
states &= ~(RADV_DYNAMIC_PATCH_CONTROL_POINTS | RADV_DYNAMIC_TESS_DOMAIN_ORIGIN);
return states;
}
static struct radv_ia_multi_vgt_param_helpers
radv_compute_ia_multi_vgt_param_helpers(struct radv_graphics_pipeline *pipeline)
{
const struct radv_physical_device *pdevice = pipeline->base.device->physical_device;
struct radv_ia_multi_vgt_param_helpers ia_multi_vgt_param = {0};
ia_multi_vgt_param.ia_switch_on_eoi = false;
if (pipeline->base.shaders[MESA_SHADER_FRAGMENT]->info.ps.prim_id_input)
ia_multi_vgt_param.ia_switch_on_eoi = true;
if (radv_pipeline_has_stage(pipeline, MESA_SHADER_GEOMETRY) && pipeline->base.shaders[MESA_SHADER_GEOMETRY]->info.uses_prim_id)
ia_multi_vgt_param.ia_switch_on_eoi = true;
if (radv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_CTRL)) {
/* SWITCH_ON_EOI must be set if PrimID is used. */
if (pipeline->base.shaders[MESA_SHADER_TESS_CTRL]->info.uses_prim_id ||
radv_get_shader(&pipeline->base, MESA_SHADER_TESS_EVAL)->info.uses_prim_id)
ia_multi_vgt_param.ia_switch_on_eoi = true;
}
ia_multi_vgt_param.partial_vs_wave = false;
if (radv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_CTRL)) {
/* Bug with tessellation and GS on Bonaire and older 2 SE chips. */
if ((pdevice->rad_info.family == CHIP_TAHITI ||
pdevice->rad_info.family == CHIP_PITCAIRN ||
pdevice->rad_info.family == CHIP_BONAIRE) &&
radv_pipeline_has_stage(pipeline, MESA_SHADER_GEOMETRY))
ia_multi_vgt_param.partial_vs_wave = true;
/* Needed for 028B6C_DISTRIBUTION_MODE != 0 */
if (pdevice->rad_info.has_distributed_tess) {
if (radv_pipeline_has_stage(pipeline, MESA_SHADER_GEOMETRY)) {
if (pdevice->rad_info.gfx_level <= GFX8)
ia_multi_vgt_param.partial_es_wave = true;
} else {
ia_multi_vgt_param.partial_vs_wave = true;
}
}
}
if (radv_pipeline_has_stage(pipeline, MESA_SHADER_GEOMETRY)) {
/* On these chips there is the possibility of a hang if the
* pipeline uses a GS and partial_vs_wave is not set.
*
* This mostly does not hit 4-SE chips, as those typically set
* ia_switch_on_eoi and then partial_vs_wave is set for pipelines
* with GS due to another workaround.
*
* Reproducer: https://bugs.freedesktop.org/show_bug.cgi?id=109242
*/
if (pdevice->rad_info.family == CHIP_TONGA ||
pdevice->rad_info.family == CHIP_FIJI ||
pdevice->rad_info.family == CHIP_POLARIS10 ||
pdevice->rad_info.family == CHIP_POLARIS11 ||
pdevice->rad_info.family == CHIP_POLARIS12 ||
pdevice->rad_info.family == CHIP_VEGAM) {
ia_multi_vgt_param.partial_vs_wave = true;
}
}
ia_multi_vgt_param.base =
/* The following field was moved to VGT_SHADER_STAGES_EN in GFX9. */
S_028AA8_MAX_PRIMGRP_IN_WAVE(pdevice->rad_info.gfx_level == GFX8 ? 2 : 0) |
S_030960_EN_INST_OPT_BASIC(pdevice->rad_info.gfx_level >= GFX9) |
S_030960_EN_INST_OPT_ADV(pdevice->rad_info.gfx_level >= GFX9);
return ia_multi_vgt_param;
}
static uint32_t
radv_get_attrib_stride(const VkPipelineVertexInputStateCreateInfo *vi, uint32_t attrib_binding)
{
for (uint32_t i = 0; i < vi->vertexBindingDescriptionCount; i++) {
const VkVertexInputBindingDescription *input_binding = &vi->pVertexBindingDescriptions[i];
if (input_binding->binding == attrib_binding)
return input_binding->stride;
}
return 0;
}
#define ALL_GRAPHICS_LIB_FLAGS \
(VK_GRAPHICS_PIPELINE_LIBRARY_VERTEX_INPUT_INTERFACE_BIT_EXT | \
VK_GRAPHICS_PIPELINE_LIBRARY_PRE_RASTERIZATION_SHADERS_BIT_EXT | \
VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT | \
VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_OUTPUT_INTERFACE_BIT_EXT)
static VkGraphicsPipelineLibraryFlagBitsEXT
shader_stage_to_pipeline_library_flags(VkShaderStageFlagBits stage)
{
assert(util_bitcount(stage) == 1);
switch (stage) {
case VK_SHADER_STAGE_VERTEX_BIT:
case VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT:
case VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT:
case VK_SHADER_STAGE_GEOMETRY_BIT:
case VK_SHADER_STAGE_TASK_BIT_EXT:
case VK_SHADER_STAGE_MESH_BIT_EXT:
return VK_GRAPHICS_PIPELINE_LIBRARY_PRE_RASTERIZATION_SHADERS_BIT_EXT;
case VK_SHADER_STAGE_FRAGMENT_BIT:
return VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT;
default:
unreachable("Invalid shader stage");
}
}
static VkResult
radv_pipeline_import_graphics_info(struct radv_graphics_pipeline *pipeline,
struct vk_graphics_pipeline_state *state,
struct radv_pipeline_layout *layout,
const VkGraphicsPipelineCreateInfo *pCreateInfo,
VkGraphicsPipelineLibraryFlagBitsEXT lib_flags)
{
RADV_FROM_HANDLE(radv_pipeline_layout, pipeline_layout, pCreateInfo->layout);
struct radv_device *device = pipeline->base.device;
VkResult result;
/* Mark all states declared dynamic at pipeline creation. */
if (pCreateInfo->pDynamicState) {
uint32_t count = pCreateInfo->pDynamicState->dynamicStateCount;
for (uint32_t s = 0; s < count; s++) {
pipeline->dynamic_states |=
radv_dynamic_state_mask(pCreateInfo->pDynamicState->pDynamicStates[s]);
}
}
/* Mark all active stages at pipeline creation. */
for (uint32_t i = 0; i < pCreateInfo->stageCount; i++) {
const VkPipelineShaderStageCreateInfo *sinfo = &pCreateInfo->pStages[i];
/* Ignore shader stages that don't need to be imported. */
if (!(shader_stage_to_pipeline_library_flags(sinfo->stage) & lib_flags))
continue;
pipeline->active_stages |= sinfo->stage;
}
result = vk_graphics_pipeline_state_fill(&device->vk, state, pCreateInfo, NULL, NULL, NULL,
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT,
&pipeline->state_data);
if (result != VK_SUCCESS)
return result;
if (lib_flags == ALL_GRAPHICS_LIB_FLAGS) {
radv_pipeline_layout_finish(device, layout);
radv_pipeline_layout_init(device, layout, false /* independent_sets */);
}
if (pipeline_layout) {
/* As explained in the specification, the application can provide a non
* compatible pipeline layout when doing optimized linking :
*
* "However, in the specific case that a final link is being
* performed between stages and
* `VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT` is specified,
* the application can override the pipeline layout with one that is
* compatible with that union but does not have the
* `VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT` flag set,
* allowing a more optimal pipeline layout to be used when
* generating the final pipeline."
*
* In that case discard whatever was imported before.
*/
if (pCreateInfo->flags & VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT &&
!pipeline_layout->independent_sets) {
radv_pipeline_layout_finish(device, layout);
radv_pipeline_layout_init(device, layout, false /* independent_sets */);
} else {
/* Otherwise if we include a layout that had independent_sets,
* propagate that property.
*/
layout->independent_sets |= pipeline_layout->independent_sets;
}
for (uint32_t s = 0; s < pipeline_layout->num_sets; s++) {
if (pipeline_layout->set[s].layout == NULL)
continue;
radv_pipeline_layout_add_set(layout, s, pipeline_layout->set[s].layout);
}
layout->push_constant_size = pipeline_layout->push_constant_size;
}
return result;
}
static void
radv_graphics_pipeline_import_lib(struct radv_graphics_pipeline *pipeline,
struct vk_graphics_pipeline_state *state,
struct radv_pipeline_layout *layout,
struct radv_graphics_lib_pipeline *lib)
{
/* There should be no common blocks between a lib we import and the current
* pipeline we're building.
*/
assert((pipeline->active_stages & lib->base.active_stages) == 0);
pipeline->dynamic_states |= lib->base.dynamic_states;
pipeline->active_stages |= lib->base.active_stages;
vk_graphics_pipeline_state_merge(state, &lib->graphics_state);
/* Import the NIR shaders (after SPIRV->NIR). */
for (uint32_t s = 0; s < ARRAY_SIZE(lib->base.base.shaders); s++) {
if (!lib->base.base.retained_shaders[s].nir)
continue;
pipeline->base.retained_shaders[s] = lib->base.base.retained_shaders[s];
}
/* Import the compiled shaders. */
for (uint32_t s = 0; s < ARRAY_SIZE(lib->base.base.shaders); s++) {
if (!lib->base.base.shaders[s])
continue;
pipeline->base.shaders[s] = radv_shader_ref(lib->base.base.shaders[s]);
/* Hold a pointer to the slab BO to indicate the shader is already uploaded. */
pipeline->base.shaders[s]->bo = lib->base.base.slab_bo;
}
/* Import the GS copy shader if present. */
if (lib->base.base.gs_copy_shader) {
assert(!pipeline->base.gs_copy_shader);
pipeline->base.gs_copy_shader = radv_shader_ref(lib->base.base.gs_copy_shader);
/* Hold a pointer to the slab BO to indicate the shader is already uploaded. */
pipeline->base.gs_copy_shader->bo = lib->base.base.slab_bo;
}
/* Refcount the slab BO to make sure it's not freed when the library is destroyed. */
if (lib->base.base.slab) {
p_atomic_inc(&lib->base.base.slab->ref_count);
}
/* Import the PS epilog if present. */
if (lib->base.ps_epilog) {
assert(!pipeline->ps_epilog);
pipeline->ps_epilog = radv_shader_part_ref(lib->base.ps_epilog);
}
/* Import the pipeline layout. */
struct radv_pipeline_layout *lib_layout = &lib->layout;
for (uint32_t s = 0; s < lib_layout->num_sets; s++) {
if (!lib_layout->set[s].layout)
continue;
radv_pipeline_layout_add_set(layout, s, lib_layout->set[s].layout);
}
layout->independent_sets = lib_layout->independent_sets;
layout->push_constant_size = MAX2(layout->push_constant_size, lib_layout->push_constant_size);
}
static void
radv_pipeline_init_input_assembly_state(struct radv_graphics_pipeline *pipeline)
{
pipeline->ia_multi_vgt_param = radv_compute_ia_multi_vgt_param_helpers(pipeline);
}
static void
radv_pipeline_init_dynamic_state(struct radv_graphics_pipeline *pipeline,
const struct vk_graphics_pipeline_state *state)
{
uint64_t needed_states = radv_pipeline_needed_dynamic_state(pipeline, state);
uint64_t states = needed_states;
pipeline->dynamic_state = default_dynamic_state;
pipeline->needed_dynamic_state = needed_states;
states &= ~pipeline->dynamic_states;
struct radv_dynamic_state *dynamic = &pipeline->dynamic_state;
if (needed_states & RADV_DYNAMIC_VIEWPORT) {
dynamic->viewport.count = state->vp->viewport_count;
if (states & RADV_DYNAMIC_VIEWPORT) {
typed_memcpy(dynamic->viewport.viewports, state->vp->viewports, state->vp->viewport_count);
for (unsigned i = 0; i < dynamic->viewport.count; i++)
radv_get_viewport_xform(&dynamic->viewport.viewports[i],
dynamic->viewport.xform[i].scale, dynamic->viewport.xform[i].translate);
}
}
if (needed_states & RADV_DYNAMIC_SCISSOR) {
dynamic->scissor.count = state->vp->scissor_count;
if (states & RADV_DYNAMIC_SCISSOR) {
typed_memcpy(dynamic->scissor.scissors, state->vp->scissors, state->vp->scissor_count);
}
}
if (states & RADV_DYNAMIC_LINE_WIDTH) {
dynamic->line_width = state->rs->line.width;
}
if (states & RADV_DYNAMIC_DEPTH_BIAS) {
dynamic->depth_bias.bias = state->rs->depth_bias.constant;
dynamic->depth_bias.clamp = state->rs->depth_bias.clamp;
dynamic->depth_bias.slope = state->rs->depth_bias.slope;
}
/* Section 9.2 of the Vulkan 1.0.15 spec says:
*
* pColorBlendState is [...] NULL if the pipeline has rasterization
* disabled or if the subpass of the render pass the pipeline is
* created against does not use any color attachments.
*/
if (states & RADV_DYNAMIC_BLEND_CONSTANTS) {
typed_memcpy(dynamic->blend_constants, state->cb->blend_constants, 4);
}
if (states & RADV_DYNAMIC_CULL_MODE) {
dynamic->cull_mode = state->rs->cull_mode;
}
if (states & RADV_DYNAMIC_FRONT_FACE) {
dynamic->front_face = state->rs->front_face;
}
if (states & RADV_DYNAMIC_PRIMITIVE_TOPOLOGY) {
dynamic->primitive_topology = si_translate_prim(state->ia->primitive_topology);
}
/* If there is no depthstencil attachment, then don't read
* pDepthStencilState. The Vulkan spec states that pDepthStencilState may
* be NULL in this case. Even if pDepthStencilState is non-NULL, there is
* no need to override the depthstencil defaults in
* radv_pipeline::dynamic_state when there is no depthstencil attachment.
*
* Section 9.2 of the Vulkan 1.0.15 spec says:
*
* pDepthStencilState is [...] NULL if the pipeline has rasterization
* disabled or if the subpass of the render pass the pipeline is created
* against does not use a depth/stencil attachment.
*/
if (needed_states && radv_pipeline_has_ds_attachments(state->rp)) {
if (states & RADV_DYNAMIC_DEPTH_BOUNDS) {
dynamic->depth_bounds.min = state->ds->depth.bounds_test.min;
dynamic->depth_bounds.max = state->ds->depth.bounds_test.max;
}
if (states & RADV_DYNAMIC_STENCIL_COMPARE_MASK) {
dynamic->stencil_compare_mask.front = state->ds->stencil.front.compare_mask;
dynamic->stencil_compare_mask.back = state->ds->stencil.back.compare_mask;
}
if (states & RADV_DYNAMIC_STENCIL_WRITE_MASK) {
dynamic->stencil_write_mask.front = state->ds->stencil.front.write_mask;
dynamic->stencil_write_mask.back = state->ds->stencil.back.write_mask;
}
if (states & RADV_DYNAMIC_STENCIL_REFERENCE) {
dynamic->stencil_reference.front = state->ds->stencil.front.reference;
dynamic->stencil_reference.back = state->ds->stencil.back.reference;
}
if (states & RADV_DYNAMIC_DEPTH_TEST_ENABLE) {
dynamic->depth_test_enable = state->ds->depth.test_enable;
}
if (states & RADV_DYNAMIC_DEPTH_WRITE_ENABLE) {
dynamic->depth_write_enable = state->ds->depth.write_enable;
}
if (states & RADV_DYNAMIC_DEPTH_COMPARE_OP) {
dynamic->depth_compare_op = state->ds->depth.compare_op;
}
if (states & RADV_DYNAMIC_DEPTH_BOUNDS_TEST_ENABLE) {
dynamic->depth_bounds_test_enable = state->ds->depth.bounds_test.enable;
}
if (states & RADV_DYNAMIC_STENCIL_TEST_ENABLE) {
dynamic->stencil_test_enable = state->ds->stencil.test_enable;
}
if (states & RADV_DYNAMIC_STENCIL_OP) {
dynamic->stencil_op.front.compare_op = state->ds->stencil.front.op.compare;
dynamic->stencil_op.front.fail_op = state->ds->stencil.front.op.fail;
dynamic->stencil_op.front.pass_op = state->ds->stencil.front.op.pass;
dynamic->stencil_op.front.depth_fail_op = state->ds->stencil.front.op.depth_fail;
dynamic->stencil_op.back.compare_op = state->ds->stencil.back.op.compare;
dynamic->stencil_op.back.fail_op = state->ds->stencil.back.op.fail;
dynamic->stencil_op.back.pass_op = state->ds->stencil.back.op.pass;
dynamic->stencil_op.back.depth_fail_op = state->ds->stencil.back.op.depth_fail;
}
}
if (needed_states & RADV_DYNAMIC_DISCARD_RECTANGLE) {
dynamic->discard_rectangle.count = state->dr->rectangle_count;
if (states & RADV_DYNAMIC_DISCARD_RECTANGLE) {
typed_memcpy(dynamic->discard_rectangle.rectangles, state->dr->rectangles,
state->dr->rectangle_count);
}
}
if (states & RADV_DYNAMIC_SAMPLE_LOCATIONS) {
unsigned count = state->ms->sample_locations->per_pixel *
state->ms->sample_locations->grid_size.width *
state->ms->sample_locations->grid_size.height;
dynamic->sample_location.per_pixel = state->ms->sample_locations->per_pixel;
dynamic->sample_location.grid_size = state->ms->sample_locations->grid_size;
dynamic->sample_location.count = count;
typed_memcpy(&dynamic->sample_location.locations[0], state->ms->sample_locations->locations,
count);
}
if (states & RADV_DYNAMIC_LINE_STIPPLE) {
dynamic->line_stipple.factor = state->rs->line.stipple.factor;
dynamic->line_stipple.pattern = state->rs->line.stipple.pattern;
}
if (states & RADV_DYNAMIC_FRAGMENT_SHADING_RATE) {
dynamic->fragment_shading_rate.size = state->fsr->fragment_size;
for (int i = 0; i < 2; i++)
dynamic->fragment_shading_rate.combiner_ops[i] = state->fsr->combiner_ops[i];
}
if (states & RADV_DYNAMIC_DEPTH_BIAS_ENABLE) {
dynamic->depth_bias_enable = state->rs->depth_bias.enable;
}
if (states & RADV_DYNAMIC_PRIMITIVE_RESTART_ENABLE) {
dynamic->primitive_restart_enable = state->ia->primitive_restart_enable;
}
if (states & RADV_DYNAMIC_RASTERIZER_DISCARD_ENABLE) {
dynamic->rasterizer_discard_enable = state->rs->rasterizer_discard_enable;
}
if (radv_pipeline_has_color_attachments(state->rp) && states & RADV_DYNAMIC_LOGIC_OP) {
if ((pipeline->dynamic_states & RADV_DYNAMIC_LOGIC_OP_ENABLE) || state->cb->logic_op_enable) {
dynamic->logic_op = si_translate_blend_logic_op(state->cb->logic_op);
} else {
dynamic->logic_op = V_028808_ROP3_COPY;
}
}
if (states & RADV_DYNAMIC_COLOR_WRITE_ENABLE) {
u_foreach_bit(i, state->cb->color_write_enables) {
dynamic->color_write_enable |= 0xfu << (i * 4);
}
}
if (states & RADV_DYNAMIC_PATCH_CONTROL_POINTS) {
dynamic->patch_control_points = state->ts->patch_control_points;
}
if (states & RADV_DYNAMIC_POLYGON_MODE) {
dynamic->polygon_mode = si_translate_fill(state->rs->polygon_mode);
}
if (states & RADV_DYNAMIC_TESS_DOMAIN_ORIGIN) {
dynamic->tess_domain_origin = state->ts->domain_origin;
}
if (radv_pipeline_has_color_attachments(state->rp) && states & RADV_DYNAMIC_LOGIC_OP_ENABLE) {
dynamic->logic_op_enable = state->cb->logic_op_enable;
}
if (states & RADV_DYNAMIC_LINE_STIPPLE_ENABLE) {
dynamic->stippled_line_enable = state->rs->line.stipple.enable;
}
if (states & RADV_DYNAMIC_ALPHA_TO_COVERAGE_ENABLE) {
dynamic->alpha_to_coverage_enable = state->ms->alpha_to_coverage_enable;
}
if (states & RADV_DYNAMIC_SAMPLE_MASK) {
dynamic->sample_mask = state->ms->sample_mask & 0xffff;
}
if (states & RADV_DYNAMIC_DEPTH_CLIP_ENABLE) {
dynamic->depth_clip_enable = state->rs->depth_clip_enable == VK_MESA_DEPTH_CLIP_ENABLE_TRUE;
}
if (states & RADV_DYNAMIC_CONSERVATIVE_RAST_MODE) {
dynamic->conservative_rast_mode = state->rs->conservative_mode;
}
if (states & RADV_DYNAMIC_DEPTH_CLIP_NEGATIVE_ONE_TO_ONE) {
dynamic->depth_clip_negative_one_to_one = state->vp->depth_clip_negative_one_to_one;
}
if (states & RADV_DYNAMIC_PROVOKING_VERTEX_MODE) {
dynamic->provoking_vertex_mode = state->rs->provoking_vertex;
}
if (states & RADV_DYNAMIC_DEPTH_CLAMP_ENABLE) {
dynamic->depth_clamp_enable = state->rs->depth_clamp_enable;
}
pipeline->dynamic_state.mask = states;
}
static bool
radv_pipeline_uses_ds_feedback_loop(const VkGraphicsPipelineCreateInfo *pCreateInfo,
const struct vk_graphics_pipeline_state *state)
{
VK_FROM_HANDLE(vk_render_pass, render_pass, state->rp->render_pass);
if (render_pass) {
uint32_t subpass_idx = state->rp->subpass;
struct vk_subpass *subpass = &render_pass->subpasses[subpass_idx];
struct vk_subpass_attachment *ds_att = subpass->depth_stencil_attachment;
for (uint32_t i = 0; i < subpass->input_count; i++) {
if (ds_att && ds_att->attachment == subpass->input_attachments[i].attachment) {
return true;
}
}
}
return (pCreateInfo->flags & VK_PIPELINE_CREATE_DEPTH_STENCIL_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT) != 0;
}
static uint32_t
radv_compute_db_shader_control(const struct radv_graphics_pipeline *pipeline,
const struct vk_graphics_pipeline_state *state,
const VkGraphicsPipelineCreateInfo *pCreateInfo)
{
const struct radv_physical_device *pdevice = pipeline->base.device->physical_device;
bool uses_ds_feedback_loop = radv_pipeline_uses_ds_feedback_loop(pCreateInfo, state);
struct radv_shader *ps = pipeline->base.shaders[MESA_SHADER_FRAGMENT];
unsigned conservative_z_export = V_02880C_EXPORT_ANY_Z;
unsigned z_order;
/* When a depth/stencil attachment is used inside feedback loops, use LATE_Z to make sure shader
* invocations read the correct value.
*/
if (!uses_ds_feedback_loop && (ps->info.ps.early_fragment_test || !ps->info.ps.writes_memory))
z_order = V_02880C_EARLY_Z_THEN_LATE_Z;
else
z_order = V_02880C_LATE_Z;
if (ps->info.ps.depth_layout == FRAG_DEPTH_LAYOUT_GREATER)
conservative_z_export = V_02880C_EXPORT_GREATER_THAN_Z;
else if (ps->info.ps.depth_layout == FRAG_DEPTH_LAYOUT_LESS)
conservative_z_export = V_02880C_EXPORT_LESS_THAN_Z;
bool disable_rbplus = pdevice->rad_info.has_rbplus && !pdevice->rad_info.rbplus_allowed;
/* It shouldn't be needed to export gl_SampleMask when MSAA is disabled
* but this appears to break Project Cars (DXVK). See
* https://bugs.freedesktop.org/show_bug.cgi?id=109401
*/
bool mask_export_enable = ps->info.ps.writes_sample_mask;
return S_02880C_Z_EXPORT_ENABLE(ps->info.ps.writes_z) |
S_02880C_STENCIL_TEST_VAL_EXPORT_ENABLE(ps->info.ps.writes_stencil) |
S_02880C_KILL_ENABLE(!!ps->info.ps.can_discard) |
S_02880C_MASK_EXPORT_ENABLE(mask_export_enable) |
S_02880C_CONSERVATIVE_Z_EXPORT(conservative_z_export) | S_02880C_Z_ORDER(z_order) |
S_02880C_DEPTH_BEFORE_SHADER(ps->info.ps.early_fragment_test) |
S_02880C_PRE_SHADER_DEPTH_COVERAGE_ENABLE(ps->info.ps.post_depth_coverage) |
S_02880C_EXEC_ON_HIER_FAIL(ps->info.ps.writes_memory) |
S_02880C_EXEC_ON_NOOP(ps->info.ps.writes_memory) |
S_02880C_DUAL_QUAD_DISABLE(disable_rbplus);
}
static struct radv_depth_stencil_state
radv_pipeline_init_depth_stencil_state(struct radv_graphics_pipeline *pipeline,
const struct vk_graphics_pipeline_state *state,
const VkGraphicsPipelineCreateInfo *pCreateInfo)
{
const struct radv_physical_device *pdevice = pipeline->base.device->physical_device;
struct radv_depth_stencil_state ds_state = {0};
bool has_depth_attachment = state->rp->depth_attachment_format != VK_FORMAT_UNDEFINED;
if (has_depth_attachment) {
/* from amdvlk: For 4xAA and 8xAA need to decompress on flush for better performance */
ds_state.db_render_override2 |=
S_028010_DECOMPRESS_Z_ON_FLUSH(state->ms && state->ms->rasterization_samples > 2);
if (pdevice->rad_info.gfx_level >= GFX10_3)
ds_state.db_render_override2 |= S_028010_CENTROID_COMPUTATION_MODE(1);
}
ds_state.db_shader_control = radv_compute_db_shader_control(pipeline, state, pCreateInfo);
if (pdevice->rad_info.gfx_level >= GFX11) {
unsigned max_allowed_tiles_in_wave = 0;
unsigned num_samples = MAX2(radv_pipeline_color_samples(state),
radv_pipeline_depth_samples(state));
if (pdevice->rad_info.has_dedicated_vram) {
if (num_samples == 8)
max_allowed_tiles_in_wave = 7;
else if (num_samples == 4)
max_allowed_tiles_in_wave = 14;
} else {
if (num_samples == 8)
max_allowed_tiles_in_wave = 8;
}
/* TODO: We may want to disable this workaround for future chips. */
if (num_samples >= 4) {
if (max_allowed_tiles_in_wave)
max_allowed_tiles_in_wave--;
else
max_allowed_tiles_in_wave = 15;
}
ds_state.db_render_control |= S_028000_OREO_MODE(V_028000_OMODE_O_THEN_B) |
S_028000_MAX_ALLOWED_TILES_IN_WAVE(max_allowed_tiles_in_wave);
}
return ds_state;
}
static void
gfx10_emit_ge_pc_alloc(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level,
uint32_t oversub_pc_lines)
{
radeon_set_uconfig_reg(
cs, R_030980_GE_PC_ALLOC,
S_030980_OVERSUB_EN(oversub_pc_lines > 0) | S_030980_NUM_PC_LINES(oversub_pc_lines - 1));
}
static void
radv_pipeline_init_gs_ring_state(struct radv_graphics_pipeline *pipeline, const struct gfx9_gs_info *gs)
{
const struct radv_physical_device *pdevice = pipeline->base.device->physical_device;
unsigned num_se = pdevice->rad_info.max_se;
unsigned wave_size = 64;
unsigned max_gs_waves = 32 * num_se; /* max 32 per SE on GCN */
/* On GFX6-GFX7, the value comes from VGT_GS_VERTEX_REUSE = 16.
* On GFX8+, the value comes from VGT_VERTEX_REUSE_BLOCK_CNTL = 30 (+2).
*/
unsigned gs_vertex_reuse = (pdevice->rad_info.gfx_level >= GFX8 ? 32 : 16) * num_se;
unsigned alignment = 256 * num_se;
/* The maximum size is 63.999 MB per SE. */
unsigned max_size = ((unsigned)(63.999 * 1024 * 1024) & ~255) * num_se;
struct radv_shader_info *gs_info = &pipeline->base.shaders[MESA_SHADER_GEOMETRY]->info;
/* Calculate the minimum size. */
unsigned min_esgs_ring_size =
align(gs->vgt_esgs_ring_itemsize * 4 * gs_vertex_reuse * wave_size, alignment);
/* These are recommended sizes, not minimum sizes. */
unsigned esgs_ring_size =
max_gs_waves * 2 * wave_size * gs->vgt_esgs_ring_itemsize * 4 * gs_info->gs.vertices_in;
unsigned gsvs_ring_size = max_gs_waves * 2 * wave_size * gs_info->gs.max_gsvs_emit_size;
min_esgs_ring_size = align(min_esgs_ring_size, alignment);
esgs_ring_size = align(esgs_ring_size, alignment);
gsvs_ring_size = align(gsvs_ring_size, alignment);
if (pdevice->rad_info.gfx_level <= GFX8)
pipeline->esgs_ring_size = CLAMP(esgs_ring_size, min_esgs_ring_size, max_size);
pipeline->gsvs_ring_size = MIN2(gsvs_ring_size, max_size);
}
struct radv_shader *
radv_get_shader(const struct radv_pipeline *pipeline, gl_shader_stage stage)
{
if (stage == MESA_SHADER_VERTEX) {
if (pipeline->shaders[MESA_SHADER_VERTEX])
return pipeline->shaders[MESA_SHADER_VERTEX];
if (pipeline->shaders[MESA_SHADER_TESS_CTRL])
return pipeline->shaders[MESA_SHADER_TESS_CTRL];
if (pipeline->shaders[MESA_SHADER_GEOMETRY])
return pipeline->shaders[MESA_SHADER_GEOMETRY];
} else if (stage == MESA_SHADER_TESS_EVAL) {
if (!pipeline->shaders[MESA_SHADER_TESS_CTRL])
return NULL;
if (pipeline->shaders[MESA_SHADER_TESS_EVAL])
return pipeline->shaders[MESA_SHADER_TESS_EVAL];
if (pipeline->shaders[MESA_SHADER_GEOMETRY])
return pipeline->shaders[MESA_SHADER_GEOMETRY];
}
return pipeline->shaders[stage];
}
static const struct radv_vs_output_info *
get_vs_output_info(const struct radv_graphics_pipeline *pipeline)
{
if (radv_pipeline_has_stage(pipeline, MESA_SHADER_GEOMETRY))
if (radv_pipeline_has_ngg(pipeline))
return &pipeline->base.shaders[MESA_SHADER_GEOMETRY]->info.outinfo;
else
return &pipeline->base.gs_copy_shader->info.outinfo;
else if (radv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_CTRL))
return &pipeline->base.shaders[MESA_SHADER_TESS_EVAL]->info.outinfo;
else if (radv_pipeline_has_stage(pipeline, MESA_SHADER_MESH))
return &pipeline->base.shaders[MESA_SHADER_MESH]->info.outinfo;
else
return &pipeline->base.shaders[MESA_SHADER_VERTEX]->info.outinfo;
}
static bool
radv_lower_viewport_to_zero(nir_shader *nir)
{
nir_function_impl *impl = nir_shader_get_entrypoint(nir);
bool progress = false;
nir_builder b;
nir_builder_init(&b, impl);
/* There should be only one deref load for VIEWPORT after lower_io_to_temporaries. */
nir_foreach_block(block, impl) {
nir_foreach_instr(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
if (intr->intrinsic != nir_intrinsic_load_deref)
continue;
nir_variable *var = nir_intrinsic_get_var(intr, 0);
if (var->data.mode != nir_var_shader_in ||
var->data.location != VARYING_SLOT_VIEWPORT)
continue;
b.cursor = nir_before_instr(instr);
nir_ssa_def_rewrite_uses(&intr->dest.ssa, nir_imm_zero(&b, 1, 32));
progress = true;
break;
}
if (progress)
break;
}
if (progress)
nir_metadata_preserve(impl, nir_metadata_block_index | nir_metadata_dominance);
else
nir_metadata_preserve(impl, nir_metadata_all);
return progress;
}
static nir_variable *
find_layer_out_var(nir_shader *nir)
{
nir_variable *var = nir_find_variable_with_location(nir, nir_var_shader_out, VARYING_SLOT_LAYER);
if (var != NULL)
return var;
var = nir_variable_create(nir, nir_var_shader_out, glsl_int_type(), "layer id");
var->data.location = VARYING_SLOT_LAYER;
var->data.interpolation = INTERP_MODE_NONE;
return var;
}
static bool
radv_should_export_multiview(const struct radv_pipeline_stage *producer,
const struct radv_pipeline_stage *consumer,
const struct radv_pipeline_key *pipeline_key)
{
/* Export the layer in the last VGT stage if multiview is used. When the next stage is unknown
* (with graphics pipeline library), the layer is exported unconditionally.
*/
return pipeline_key->has_multiview_view_index &&
(!consumer || consumer->stage == MESA_SHADER_FRAGMENT) &&
!(producer->nir->info.outputs_written & VARYING_BIT_LAYER);
}
static bool
radv_export_multiview(nir_shader *nir)
{
nir_function_impl *impl = nir_shader_get_entrypoint(nir);
bool progress = false;
nir_builder b;
nir_builder_init(&b, impl);
/* This pass is not suitable for mesh shaders, because it can't know the mapping between API mesh
* shader invocations and output primitives. Needs to be handled in ac_nir_lower_ngg.
*/
assert(nir->info.stage == MESA_SHADER_VERTEX ||
nir->info.stage == MESA_SHADER_TESS_EVAL ||
nir->info.stage == MESA_SHADER_GEOMETRY);
/* Iterate in reverse order since there should be only one deref store to POS after
* lower_io_to_temporaries for vertex shaders and inject the layer there. For geometry shaders,
* the layer is injected right before every emit_vertex_with_counter.
*/
nir_variable *layer = NULL;
nir_foreach_block_reverse(block, impl) {
nir_foreach_instr_reverse(instr, block) {
if (instr->type != nir_instr_type_intrinsic)
continue;
if (nir->info.stage == MESA_SHADER_GEOMETRY) {
nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
if (intr->intrinsic != nir_intrinsic_emit_vertex_with_counter)
continue;
b.cursor = nir_before_instr(instr);
} else {
nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
if (intr->intrinsic != nir_intrinsic_store_deref)
continue;
nir_variable *var = nir_intrinsic_get_var(intr, 0);
if (var->data.mode != nir_var_shader_out || var->data.location != VARYING_SLOT_POS)
continue;
b.cursor = nir_after_instr(instr);
}
if (!layer)
layer = find_layer_out_var(nir);
nir_store_var(&b, layer, nir_load_view_index(&b), 1);
/* Update outputs_written to reflect that the pass added a new output. */
nir->info.outputs_written |= BITFIELD64_BIT(VARYING_SLOT_LAYER);
progress = true;
if (nir->info.stage == MESA_SHADER_VERTEX)
break;
}
if (nir->info.stage == MESA_SHADER_VERTEX && progress)
break;
}
if (progress)
nir_metadata_preserve(impl, nir_metadata_block_index | nir_metadata_dominance);
else
nir_metadata_preserve(impl, nir_metadata_all);
return progress;
}
static bool
radv_should_export_implicit_primitive_id(const struct radv_pipeline_stage *producer,
const struct radv_pipeline_stage *consumer)
{
/* When the primitive ID is read by FS, we must ensure that it's exported by the previous vertex
* stage because it's implicit for VS or TES (but required by the Vulkan spec for GS or MS).
*
* There is two situations to handle:
* - when the next stage is unknown (with graphics pipeline library), the primitive ID is
* exported unconditionally
* - when the pipeline uses NGG, the primitive ID is exported during NGG lowering
*/
assert(producer->stage == MESA_SHADER_VERTEX || producer->stage == MESA_SHADER_TESS_EVAL);
if ((producer->nir->info.outputs_written & VARYING_BIT_PRIMITIVE_ID) || producer->info.is_ngg)
return false;
return !consumer || (consumer->stage == MESA_SHADER_FRAGMENT &&
(consumer->nir->info.inputs_read & VARYING_BIT_PRIMITIVE_ID));
}
static bool
radv_export_implicit_primitive_id(nir_shader *nir)
{
nir_function_impl *impl = nir_shader_get_entrypoint(nir);
nir_builder b;
nir_builder_init(&b, impl);
b.cursor = nir_after_cf_list(&impl->body);
nir_variable *var = nir_variable_create(nir, nir_var_shader_out, glsl_int_type(), NULL);
var->data.location = VARYING_SLOT_PRIMITIVE_ID;
var->data.interpolation = INTERP_MODE_NONE;
nir_store_var(&b, var, nir_load_primitive_id(&b), 1);
/* Update outputs_written to reflect that the pass added a new output. */
nir->info.outputs_written |= BITFIELD64_BIT(VARYING_SLOT_PRIMITIVE_ID);
nir_metadata_preserve(impl, nir_metadata_block_index | nir_metadata_dominance);
return true;
}
static void
radv_remove_point_size(const struct radv_pipeline_key *pipeline_key,
nir_shader *producer, nir_shader *consumer)
{
if ((consumer->info.inputs_read & VARYING_BIT_PSIZ) ||
!(producer->info.outputs_written & VARYING_BIT_PSIZ))
return;
/* Do not remove PSIZ if the shader uses XFB because it might be stored. */
if (producer->xfb_info)
return;
/* Do not remove PSIZ if the rasterization primitive uses points. */
if (consumer->info.stage == MESA_SHADER_FRAGMENT &&
((producer->info.stage == MESA_SHADER_TESS_EVAL && producer->info.tess.point_mode) ||
(producer->info.stage == MESA_SHADER_GEOMETRY &&
producer->info.gs.output_primitive == SHADER_PRIM_POINTS) ||
(producer->info.stage == MESA_SHADER_MESH &&
producer->info.mesh.primitive_type == SHADER_PRIM_POINTS)))
return;
nir_variable *var =
nir_find_variable_with_location(producer, nir_var_shader_out, VARYING_SLOT_PSIZ);
assert(var);
/* Change PSIZ to a global variable which allows it to be DCE'd. */
var->data.location = 0;
var->data.mode = nir_var_shader_temp;
producer->info.outputs_written &= ~VARYING_BIT_PSIZ;
NIR_PASS_V(producer, nir_fixup_deref_modes);
NIR_PASS(_, producer, nir_remove_dead_variables, nir_var_shader_temp, NULL);
NIR_PASS(_, producer, nir_opt_dce);
}
static void
radv_remove_color_exports(const struct radv_pipeline_key *pipeline_key, nir_shader *nir)
{
bool fixup_derefs = false;
nir_foreach_shader_out_variable(var, nir) {
int idx = var->data.location;
idx -= FRAG_RESULT_DATA0;
if (idx < 0)
continue;
unsigned col_format = (pipeline_key->ps.col_format >> (4 * idx)) & 0xf;
unsigned cb_target_mask = (pipeline_key->ps.cb_target_mask >> (4 * idx)) & 0xf;
if (col_format == V_028714_SPI_SHADER_ZERO ||
(col_format == V_028714_SPI_SHADER_32_R && !cb_target_mask &&
!pipeline_key->ps.mrt0_is_dual_src)) {
/* Remove the color export if it's unused or in presence of holes. */
nir->info.outputs_written &= ~BITFIELD64_BIT(var->data.location);
var->data.location = 0;
var->data.mode = nir_var_shader_temp;
fixup_derefs = true;
}
}
if (fixup_derefs) {
NIR_PASS_V(nir, nir_fixup_deref_modes);
NIR_PASS(_, nir, nir_remove_dead_variables, nir_var_shader_temp, NULL);
NIR_PASS(_, nir, nir_opt_dce);
}
}
static void
merge_tess_info(struct shader_info *tes_info, struct shader_info *tcs_info)
{
/* The Vulkan 1.0.38 spec, section 21.1 Tessellator says:
*
* "PointMode. Controls generation of points rather than triangles
* or lines. This functionality defaults to disabled, and is
* enabled if either shader stage includes the execution mode.
*
* and about Triangles, Quads, IsoLines, VertexOrderCw, VertexOrderCcw,
* PointMode, SpacingEqual, SpacingFractionalEven, SpacingFractionalOdd,
* and OutputVertices, it says:
*
* "One mode must be set in at least one of the tessellation
* shader stages."
*
* So, the fields can be set in either the TCS or TES, but they must
* agree if set in both. Our backend looks at TES, so bitwise-or in
* the values from the TCS.
*/
assert(tcs_info->tess.tcs_vertices_out == 0 || tes_info->tess.tcs_vertices_out == 0 ||
tcs_info->tess.tcs_vertices_out == tes_info->tess.tcs_vertices_out);
tes_info->tess.tcs_vertices_out |= tcs_info->tess.tcs_vertices_out;
assert(tcs_info->tess.spacing == TESS_SPACING_UNSPECIFIED ||
tes_info->tess.spacing == TESS_SPACING_UNSPECIFIED ||
tcs_info->tess.spacing == tes_info->tess.spacing);
tes_info->tess.spacing |= tcs_info->tess.spacing;
assert(tcs_info->tess._primitive_mode == TESS_PRIMITIVE_UNSPECIFIED ||
tes_info->tess._primitive_mode == TESS_PRIMITIVE_UNSPECIFIED ||
tcs_info->tess._primitive_mode == tes_info->tess._primitive_mode);
tes_info->tess._primitive_mode |= tcs_info->tess._primitive_mode;
tes_info->tess.ccw |= tcs_info->tess.ccw;
tes_info->tess.point_mode |= tcs_info->tess.point_mode;
/* Copy the merged info back to the TCS */
tcs_info->tess.tcs_vertices_out = tes_info->tess.tcs_vertices_out;
tcs_info->tess.spacing = tes_info->tess.spacing;
tcs_info->tess._primitive_mode = tes_info->tess._primitive_mode;
tcs_info->tess.ccw = tes_info->tess.ccw;
tcs_info->tess.point_mode = tes_info->tess.point_mode;
}
static void
radv_lower_io_to_scalar_early(nir_shader *nir, nir_variable_mode mask)
{
bool progress = false;
NIR_PASS(progress, nir, nir_lower_array_deref_of_vec, mask,
nir_lower_direct_array_deref_of_vec_load | nir_lower_indirect_array_deref_of_vec_load |
nir_lower_direct_array_deref_of_vec_store |
nir_lower_indirect_array_deref_of_vec_store);
NIR_PASS(progress, nir, nir_lower_io_to_scalar_early, mask);
if (progress) {
/* Optimize the new vector code and then remove dead vars */
NIR_PASS(_, nir, nir_copy_prop);
NIR_PASS(_, nir, nir_opt_shrink_vectors);
if (mask & nir_var_shader_out) {
/* Optimize swizzled movs of load_const for nir_link_opt_varyings's constant propagation. */
NIR_PASS(_, nir, nir_opt_constant_folding);
/* For nir_link_opt_varyings's duplicate input opt */
NIR_PASS(_, nir, nir_opt_cse);
}
/* Run copy-propagation to help remove dead output variables (some shaders have useless copies
* to/from an output), so compaction later will be more effective.
*
* This will have been done earlier but it might not have worked because the outputs were
* vector.
*/
if (nir->info.stage == MESA_SHADER_TESS_CTRL)
NIR_PASS(_, nir, nir_opt_copy_prop_vars);
NIR_PASS(_, nir, nir_opt_dce);
NIR_PASS(_, nir, nir_remove_dead_variables,
nir_var_function_temp | nir_var_shader_in | nir_var_shader_out, NULL);
}
}
static void
radv_pipeline_link_shaders(const struct radv_device *device,
nir_shader *producer, nir_shader *consumer,
const struct radv_pipeline_key *pipeline_key)
{
const enum amd_gfx_level gfx_level = device->physical_device->rad_info.gfx_level;
bool progress;
if (consumer->info.stage == MESA_SHADER_FRAGMENT) {
/* Lower the viewport index to zero when the last vertex stage doesn't export it. */
if ((consumer->info.inputs_read & VARYING_BIT_VIEWPORT) &&
!(producer->info.outputs_written & VARYING_BIT_VIEWPORT)) {
NIR_PASS(_, consumer, radv_lower_viewport_to_zero);
}
/* Lower the view index to map on the layer. */
NIR_PASS(_, consumer, radv_lower_view_index, producer->info.stage == MESA_SHADER_MESH);
}
if (pipeline_key->optimisations_disabled)
return;
if (consumer->info.stage == MESA_SHADER_FRAGMENT &&
producer->info.has_transform_feedback_varyings) {
nir_link_xfb_varyings(producer, consumer);
}
nir_lower_io_arrays_to_elements(producer, consumer);
nir_validate_shader(producer, "after nir_lower_io_arrays_to_elements");
nir_validate_shader(consumer, "after nir_lower_io_arrays_to_elements");
radv_lower_io_to_scalar_early(producer, nir_var_shader_out);
radv_lower_io_to_scalar_early(consumer, nir_var_shader_in);
/* Remove PSIZ from shaders when it's not needed.
* This is typically produced by translation layers like Zink or D9VK.
*/
if (pipeline_key->enable_remove_point_size)
radv_remove_point_size(pipeline_key, producer, consumer);
if (nir_link_opt_varyings(producer, consumer)) {
nir_validate_shader(producer, "after nir_link_opt_varyings");
nir_validate_shader(consumer, "after nir_link_opt_varyings");
NIR_PASS(_, consumer, nir_opt_constant_folding);
NIR_PASS(_, consumer, nir_opt_algebraic);
NIR_PASS(_, consumer, nir_opt_dce);
}
NIR_PASS(_, producer, nir_remove_dead_variables, nir_var_shader_out, NULL);
NIR_PASS(_, consumer, nir_remove_dead_variables, nir_var_shader_in, NULL);
progress = nir_remove_unused_varyings(producer, consumer);
nir_compact_varyings(producer, consumer, true);
/* nir_compact_varyings changes deleted varyings into shader_temp.
* We need to remove these otherwise we risk them being lowered to scratch.
* This can especially happen to arrayed outputs.
*/
NIR_PASS(_, producer, nir_remove_dead_variables, nir_var_shader_temp, NULL);
NIR_PASS(_, consumer, nir_remove_dead_variables, nir_var_shader_temp, NULL);
nir_validate_shader(producer, "after nir_compact_varyings");
nir_validate_shader(consumer, "after nir_compact_varyings");
if (producer->info.stage == MESA_SHADER_MESH) {
/* nir_compact_varyings can change the location of per-vertex and per-primitive outputs */
nir_shader_gather_info(producer, nir_shader_get_entrypoint(producer));
}
const bool has_geom_or_tess = consumer->info.stage == MESA_SHADER_GEOMETRY ||
consumer->info.stage == MESA_SHADER_TESS_CTRL;
const bool merged_gs = consumer->info.stage == MESA_SHADER_GEOMETRY && gfx_level >= GFX9;
if (producer->info.stage == MESA_SHADER_TESS_CTRL ||
producer->info.stage == MESA_SHADER_MESH ||
(producer->info.stage == MESA_SHADER_VERTEX && has_geom_or_tess) ||
(producer->info.stage == MESA_SHADER_TESS_EVAL && merged_gs)) {
NIR_PASS(_, producer, nir_lower_io_to_vector, nir_var_shader_out);
if (producer->info.stage == MESA_SHADER_TESS_CTRL)
NIR_PASS(_, producer, nir_vectorize_tess_levels);
NIR_PASS(_, producer, nir_opt_combine_stores, nir_var_shader_out);
}
if (consumer->info.stage == MESA_SHADER_GEOMETRY ||
consumer->info.stage == MESA_SHADER_TESS_CTRL ||
consumer->info.stage == MESA_SHADER_TESS_EVAL) {
NIR_PASS(_, consumer, nir_lower_io_to_vector, nir_var_shader_in);
}
if (progress) {
progress = false;
NIR_PASS(progress, producer, nir_lower_global_vars_to_local);
if (progress) {
ac_nir_lower_indirect_derefs(producer, gfx_level);
/* remove dead writes, which can remove input loads */
NIR_PASS(_, producer, nir_lower_vars_to_ssa);
NIR_PASS(_, producer, nir_opt_dce);
}
progress = false;
NIR_PASS(progress, consumer, nir_lower_global_vars_to_local);
if (progress) {
ac_nir_lower_indirect_derefs(consumer, gfx_level);
}
}
}
static const gl_shader_stage graphics_shader_order[] = {
MESA_SHADER_VERTEX,
MESA_SHADER_TESS_CTRL,
MESA_SHADER_TESS_EVAL,
MESA_SHADER_GEOMETRY,
MESA_SHADER_TASK,
MESA_SHADER_MESH,
MESA_SHADER_FRAGMENT,
};
static void
radv_pipeline_link_vs(const struct radv_device *device, struct radv_pipeline_stage *vs_stage,
struct radv_pipeline_stage *next_stage,
const struct radv_pipeline_key *pipeline_key)
{
assert(vs_stage->nir->info.stage == MESA_SHADER_VERTEX);
if (radv_should_export_implicit_primitive_id(vs_stage, next_stage)) {
NIR_PASS(_, vs_stage->nir, radv_export_implicit_primitive_id);
}
if (radv_should_export_multiview(vs_stage, next_stage, pipeline_key)) {
NIR_PASS(_, vs_stage->nir, radv_export_multiview);
}
if (next_stage) {
assert(next_stage->nir->info.stage == MESA_SHADER_TESS_CTRL ||
next_stage->nir->info.stage == MESA_SHADER_GEOMETRY ||
next_stage->nir->info.stage == MESA_SHADER_FRAGMENT);
radv_pipeline_link_shaders(device, vs_stage->nir, next_stage->nir, pipeline_key);
}
nir_foreach_shader_in_variable(var, vs_stage->nir) {
var->data.driver_location = var->data.location;
}
if (next_stage && next_stage->nir->info.stage == MESA_SHADER_TESS_CTRL) {
nir_linked_io_var_info vs2tcs =
nir_assign_linked_io_var_locations(vs_stage->nir, next_stage->nir);
vs_stage->info.vs.num_linked_outputs = vs2tcs.num_linked_io_vars;
next_stage->info.tcs.num_linked_inputs = vs2tcs.num_linked_io_vars;
} else if (next_stage && next_stage->nir->info.stage == MESA_SHADER_GEOMETRY) {
nir_linked_io_var_info vs2gs =
nir_assign_linked_io_var_locations(vs_stage->nir, next_stage->nir);
vs_stage->info.vs.num_linked_outputs = vs2gs.num_linked_io_vars;
next_stage->info.gs.num_linked_inputs = vs2gs.num_linked_io_vars;
} else {
nir_foreach_shader_out_variable(var, vs_stage->nir) {
var->data.driver_location = var->data.location;
}
}
}
static void
radv_pipeline_link_tcs(const struct radv_device *device, struct radv_pipeline_stage *tcs_stage,
struct radv_pipeline_stage *tes_stage,
const struct radv_pipeline_key *pipeline_key)
{
assert(tcs_stage->nir->info.stage == MESA_SHADER_TESS_CTRL);
assert(tes_stage->nir->info.stage == MESA_SHADER_TESS_EVAL);
radv_pipeline_link_shaders(device, tcs_stage->nir, tes_stage->nir, pipeline_key);
nir_lower_patch_vertices(tes_stage->nir, tcs_stage->nir->info.tess.tcs_vertices_out, NULL);
/* Copy TCS info into the TES info */
merge_tess_info(&tes_stage->nir->info, &tcs_stage->nir->info);
nir_linked_io_var_info tcs2tes =
nir_assign_linked_io_var_locations(tcs_stage->nir, tes_stage->nir);
tcs_stage->info.tcs.num_linked_outputs = tcs2tes.num_linked_io_vars;
tcs_stage->info.tcs.num_linked_patch_outputs = tcs2tes.num_linked_patch_io_vars;
tes_stage->info.tes.num_linked_inputs = tcs2tes.num_linked_io_vars;
tes_stage->info.tes.num_linked_patch_inputs = tcs2tes.num_linked_patch_io_vars;
}
static void
radv_pipeline_link_tes(const struct radv_device *device, struct radv_pipeline_stage *tes_stage,
struct radv_pipeline_stage *next_stage,
const struct radv_pipeline_key *pipeline_key)
{
assert(tes_stage->nir->info.stage == MESA_SHADER_TESS_EVAL);
if (radv_should_export_implicit_primitive_id(tes_stage, next_stage)) {
NIR_PASS(_, tes_stage->nir, radv_export_implicit_primitive_id);
}
if (radv_should_export_multiview(tes_stage, next_stage, pipeline_key)) {
NIR_PASS(_, tes_stage->nir, radv_export_multiview);
}
if (next_stage) {
assert(next_stage->nir->info.stage == MESA_SHADER_GEOMETRY ||
next_stage->nir->info.stage == MESA_SHADER_FRAGMENT);
radv_pipeline_link_shaders(device, tes_stage->nir, next_stage->nir, pipeline_key);
}
if (next_stage && next_stage->nir->info.stage == MESA_SHADER_GEOMETRY) {
nir_linked_io_var_info tes2gs =
nir_assign_linked_io_var_locations(tes_stage->nir, next_stage->nir);
tes_stage->info.tes.num_linked_outputs = tes2gs.num_linked_io_vars;
next_stage->info.gs.num_linked_inputs = tes2gs.num_linked_io_vars;
} else {
nir_foreach_shader_out_variable(var, tes_stage->nir) {
var->data.driver_location = var->data.location;
}
}
}
static void
radv_pipeline_link_gs(const struct radv_device *device, struct radv_pipeline_stage *gs_stage,
struct radv_pipeline_stage *fs_stage,
const struct radv_pipeline_key *pipeline_key)
{
assert(gs_stage->nir->info.stage == MESA_SHADER_GEOMETRY);
if (radv_should_export_multiview(gs_stage, fs_stage, pipeline_key)) {
NIR_PASS(_, gs_stage->nir, radv_export_multiview);
}
if (fs_stage) {
assert(fs_stage->nir->info.stage == MESA_SHADER_FRAGMENT);
radv_pipeline_link_shaders(device, gs_stage->nir, fs_stage->nir, pipeline_key);
}
nir_foreach_shader_out_variable(var, gs_stage->nir) {
var->data.driver_location = var->data.location;
}
}
static void
radv_pipeline_link_task(const struct radv_device *device, struct radv_pipeline_stage *task_stage,
struct radv_pipeline_stage *mesh_stage,
const struct radv_pipeline_key *pipeline_key)
{
assert(task_stage->nir->info.stage == MESA_SHADER_TASK);
assert(mesh_stage->nir->info.stage == MESA_SHADER_MESH);
/* Linking task and mesh shaders shouldn't do anything for now but keep it for consistency. */
radv_pipeline_link_shaders(device, task_stage->nir, mesh_stage->nir, pipeline_key);
}
static void
radv_pipeline_link_mesh(const struct radv_device *device, struct radv_pipeline_stage *mesh_stage,
struct radv_pipeline_stage *fs_stage,
const struct radv_pipeline_key *pipeline_key)
{
assert(mesh_stage->nir->info.stage == MESA_SHADER_MESH);
if (fs_stage) {
assert(fs_stage->nir->info.stage == MESA_SHADER_FRAGMENT);
nir_foreach_shader_in_variable(var, fs_stage->nir) {
/* These variables are per-primitive when used with a mesh shader. */
if (var->data.location == VARYING_SLOT_PRIMITIVE_ID ||
var->data.location == VARYING_SLOT_VIEWPORT ||
var->data.location == VARYING_SLOT_LAYER) {
var->data.per_primitive = true;
}
}
radv_pipeline_link_shaders(device, mesh_stage->nir, fs_stage->nir, pipeline_key);
}
/* ac_nir_lower_ngg ignores driver locations for mesh shaders, but set them to all zero just to
* be on the safe side.
*/
nir_foreach_shader_out_variable(var, mesh_stage->nir) {
var->data.driver_location = 0;
}
}
static void
radv_pipeline_link_fs(struct radv_pipeline_stage *fs_stage,
const struct radv_pipeline_key *pipeline_key)
{
assert(fs_stage->nir->info.stage == MESA_SHADER_FRAGMENT);
if (!pipeline_key->ps.has_epilog) {
/* Only remove color exports when the format is known. */
radv_remove_color_exports(pipeline_key, fs_stage->nir);
}
nir_foreach_shader_out_variable(var, fs_stage->nir) {
var->data.driver_location = var->data.location + var->data.index;
}
}
static void
radv_graphics_pipeline_link(const struct radv_pipeline *pipeline,
const struct radv_pipeline_key *pipeline_key,
struct radv_pipeline_stage *stages)
{
const struct radv_device *device = pipeline->device;
/* Walk backwards to link */
struct radv_pipeline_stage *next_stage = NULL;
for (int i = ARRAY_SIZE(graphics_shader_order) - 1; i >= 0; i--) {
gl_shader_stage s = graphics_shader_order[i];
if (!stages[s].nir)
continue;
switch (s) {
case MESA_SHADER_VERTEX:
radv_pipeline_link_vs(device, &stages[s], next_stage, pipeline_key);
break;
case MESA_SHADER_TESS_CTRL:
radv_pipeline_link_tcs(device, &stages[s], next_stage, pipeline_key);
break;
case MESA_SHADER_TESS_EVAL:
radv_pipeline_link_tes(device, &stages[s], next_stage, pipeline_key);
break;
case MESA_SHADER_GEOMETRY:
radv_pipeline_link_gs(device, &stages[s], next_stage, pipeline_key);
break;
case MESA_SHADER_TASK:
radv_pipeline_link_task(device, &stages[s], next_stage, pipeline_key);
break;
case MESA_SHADER_MESH:
radv_pipeline_link_mesh(device, &stages[s], next_stage, pipeline_key);
break;
case MESA_SHADER_FRAGMENT:
radv_pipeline_link_fs(&stages[s], pipeline_key);
break;
default:
unreachable("Invalid graphics shader stage");
}
next_stage = &stages[s];
}
}
struct radv_pipeline_key
radv_generate_pipeline_key(const struct radv_pipeline *pipeline, VkPipelineCreateFlags flags)
{
struct radv_device *device = pipeline->device;
struct radv_pipeline_key key;
memset(&key, 0, sizeof(key));
if (flags & VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT)
key.optimisations_disabled = 1;
key.disable_aniso_single_level = device->instance->disable_aniso_single_level &&
device->physical_device->rad_info.gfx_level < GFX8;
key.image_2d_view_of_3d = device->image_2d_view_of_3d &&
device->physical_device->rad_info.gfx_level == GFX9;
return key;
}
static struct radv_pipeline_key
radv_generate_graphics_pipeline_key(const struct radv_graphics_pipeline *pipeline,
const VkGraphicsPipelineCreateInfo *pCreateInfo,
const struct vk_graphics_pipeline_state *state,
const struct radv_blend_state *blend)
{
struct radv_device *device = pipeline->base.device;
const struct radv_physical_device *pdevice = device->physical_device;
struct radv_pipeline_key key = radv_generate_pipeline_key(&pipeline->base, pCreateInfo->flags);
key.has_multiview_view_index = state->rp ? !!state->rp->view_mask : 0;
if (pipeline->dynamic_states & RADV_DYNAMIC_VERTEX_INPUT) {
key.vs.has_prolog = true;
}
/* Vertex input state */
if (state->vi) {
u_foreach_bit(i, state->vi->attributes_valid) {
uint32_t binding =