Google Git
Sign in
fuchsia / third_party / mesa / 5f8d56bab6ec77b069df6023a3944dd3f5fb6625 / . / src / panfrost / bifrost / bi_schedule.c
blob: 877d57aaadf36e8e8d75c3d31bbbe86089ea0ca3 [file] [log] [blame]
/*
* Copyright (C) 2020 Collabora Ltd.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* Authors (Collabora):
* Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
*/
#include "compiler.h"
#include "bi_builder.h"
/* Arguments common to worklist, passed by value for convenience */
struct bi_worklist {
/* # of instructions in the block */
unsigned count;
/* Instructions in the block */
bi_instr **instructions;
/* Bitset of instructions in the block ready for scheduling */
BITSET_WORD *worklist;
/* The backwards dependency graph. nr_dependencies is the number of
* unscheduled instructions that must still be scheduled after (before)
* this instruction. dependents are which instructions need to be
* scheduled before (after) this instruction. */
unsigned *dep_counts;
BITSET_WORD **dependents;
};
/* State of a single tuple and clause under construction */
struct bi_reg_state {
/* Number of register writes */
unsigned nr_writes;
/* Register reads, expressed as (equivalence classes of)
* sources. Only 3 reads are allowed, but up to 2 may spill as
* "forced" for the next scheduled tuple, provided such a tuple
* can be constructed */
bi_index reads[5];
unsigned nr_reads;
/* The previous tuple scheduled (= the next tuple executed in the
* program) may require certain writes, in order to bypass the register
* file and use a temporary passthrough for the value. Up to 2 such
* constraints are architecturally satisfiable */
unsigned forced_count;
bi_index forceds[2];
};
struct bi_tuple_state {
/* Is this the last tuple in the clause */
bool last;
/* Scheduled ADD instruction, or null if none */
bi_instr *add;
/* Reads for previous (succeeding) tuple */
bi_index prev_reads[5];
unsigned nr_prev_reads;
bi_tuple *prev;
/* Register slot state for current tuple */
struct bi_reg_state reg;
/* Constants are shared in the tuple. If constant_count is nonzero, it
* is a size for constant count. Otherwise, fau is the slot read from
* FAU, or zero if none is assigned. Ordinarily FAU slot 0 reads zero,
* but within a tuple, that should be encoded as constant_count != 0
* and constants[0] = constants[1] = 0 */
unsigned constant_count;
union {
uint32_t constants[2];
enum bir_fau fau;
};
unsigned pcrel_idx;
};
struct bi_const_state {
unsigned constant_count;
bool pcrel; /* applies to first const */
uint32_t constants[2];
/* Index of the constant into the clause */
unsigned word_idx;
};
enum bi_ftz_state {
/* No flush-to-zero state assigned yet */
BI_FTZ_STATE_NONE,
/* Never flush-to-zero */
BI_FTZ_STATE_DISABLE,
/* Always flush-to-zero */
BI_FTZ_STATE_ENABLE,
};
/* At this point, pseudoinstructions have been lowered so sources/destinations
* are limited to what's physically supported.
*/
#define BI_MAX_PHYS_SRCS 4
#define BI_MAX_PHYS_DESTS 2
struct bi_clause_state {
/* Has a message-passing instruction already been assigned? */
bool message;
/* Indices already accessed, this needs to be tracked to avoid hazards
* around message-passing instructions */
unsigned access_count;
bi_index accesses[(BI_MAX_PHYS_SRCS + BI_MAX_PHYS_DESTS) * 16];
unsigned tuple_count;
struct bi_const_state consts[8];
/* Numerical state of the clause */
enum bi_ftz_state ftz;
};
/* Determines messsage type by checking the table and a few special cases. Only
* case missing is tilebuffer instructions that access depth/stencil, which
* require a Z_STENCIL message (to implement
* ARM_shader_framebuffer_fetch_depth_stencil) */
static enum bifrost_message_type
bi_message_type_for_instr(bi_instr *ins)
{
enum bifrost_message_type msg = bi_opcode_props[ins->op].message;
bool ld_var_special = (ins->op == BI_OPCODE_LD_VAR_SPECIAL);
if (ld_var_special && ins->varying_name == BI_VARYING_NAME_FRAG_Z)
return BIFROST_MESSAGE_Z_STENCIL;
if (msg == BIFROST_MESSAGE_LOAD && ins->seg == BI_SEG_UBO)
return BIFROST_MESSAGE_ATTRIBUTE;
return msg;
}
/* Attribute, texture, and UBO load (attribute message) instructions support
* bindless, so just check the message type */
ASSERTED static bool
bi_supports_dtsel(bi_instr *ins)
{
switch (bi_message_type_for_instr(ins)) {
case BIFROST_MESSAGE_ATTRIBUTE:
return ins->op != BI_OPCODE_LD_GCLK_U64;
case BIFROST_MESSAGE_TEX:
return true;
default:
return false;
}
}
/* Adds an edge to the dependency graph */
static void
bi_push_dependency(unsigned parent, unsigned child,
BITSET_WORD **dependents, unsigned *dep_counts)
{
if (!BITSET_TEST(dependents[parent], child)) {
BITSET_SET(dependents[parent], child);
dep_counts[child]++;
}
}
static void
add_dependency(struct util_dynarray *table, unsigned index, unsigned child,
BITSET_WORD **dependents, unsigned *dep_counts)
{
assert(index < 64);
util_dynarray_foreach(table + index, unsigned, parent)
bi_push_dependency(*parent, child, dependents, dep_counts);
}
static void
mark_access(struct util_dynarray *table, unsigned index, unsigned parent)
{
assert(index < 64);
util_dynarray_append(&table[index], unsigned, parent);
}
static bool
bi_is_sched_barrier(bi_instr *I)
{
switch (I->op) {
case BI_OPCODE_BARRIER:
case BI_OPCODE_DISCARD_F32:
return true;
default:
return false;
}
}
static void
bi_create_dependency_graph(struct bi_worklist st, bool inorder, bool is_blend)
{
struct util_dynarray last_read[64], last_write[64];
for (unsigned i = 0; i < 64; ++i) {
util_dynarray_init(&last_read[i], NULL);
util_dynarray_init(&last_write[i], NULL);
}
/* Initialize dependency graph */
for (unsigned i = 0; i < st.count; ++i) {
st.dependents[i] =
calloc(BITSET_WORDS(st.count), sizeof(BITSET_WORD));
st.dep_counts[i] = 0;
}
unsigned prev_msg = ~0;
/* Populate dependency graph */
for (signed i = st.count - 1; i >= 0; --i) {
bi_instr *ins = st.instructions[i];
bi_foreach_src(ins, s) {
if (ins->src[s].type != BI_INDEX_REGISTER) continue;
unsigned count = bi_count_read_registers(ins, s);
for (unsigned c = 0; c < count; ++c)
add_dependency(last_write, ins->src[s].value + c, i, st.dependents, st.dep_counts);
}
/* Keep message-passing ops in order. (This pass only cares
* about bundling; reordering of message-passing instructions
* happens during earlier scheduling.) */
if (bi_message_type_for_instr(ins)) {
if (prev_msg != ~0)
bi_push_dependency(prev_msg, i, st.dependents, st.dep_counts);
prev_msg = i;
}
/* Handle schedule barriers, adding All the deps */
if (inorder || bi_is_sched_barrier(ins)) {
for (unsigned j = 0; j < st.count; ++j) {
if (i == j) continue;
bi_push_dependency(MAX2(i, j), MIN2(i, j),
st.dependents, st.dep_counts);
}
}
bi_foreach_dest(ins, d) {
assert(ins->dest[d].type == BI_INDEX_REGISTER);
unsigned dest = ins->dest[d].value;
unsigned count = bi_count_write_registers(ins, d);
for (unsigned c = 0; c < count; ++c) {
add_dependency(last_read, dest + c, i, st.dependents, st.dep_counts);
add_dependency(last_write, dest + c, i, st.dependents, st.dep_counts);
mark_access(last_write, dest + c, i);
}
}
/* Blend shaders are allowed to clobber R0-R15. Treat these
* registers like extra destinations for scheduling purposes.
*/
if (ins->op == BI_OPCODE_BLEND && !is_blend) {
for (unsigned c = 0; c < 16; ++c) {
add_dependency(last_read, c, i, st.dependents, st.dep_counts);
add_dependency(last_write, c, i, st.dependents, st.dep_counts);
mark_access(last_write, c, i);
}
}
bi_foreach_src(ins, s) {
if (ins->src[s].type != BI_INDEX_REGISTER) continue;
unsigned count = bi_count_read_registers(ins, s);
for (unsigned c = 0; c < count; ++c)
mark_access(last_read, ins->src[s].value + c, i);
}
}
/* If there is a branch, all instructions depend on it, as interblock
* execution must be purely in-order */
bi_instr *last = st.instructions[st.count - 1];
if (last->branch_target || last->op == BI_OPCODE_JUMP) {
for (signed i = st.count - 2; i >= 0; --i)
bi_push_dependency(st.count - 1, i, st.dependents, st.dep_counts);
}
/* Free the intermediate structures */
for (unsigned i = 0; i < 64; ++i) {
util_dynarray_fini(&last_read[i]);
util_dynarray_fini(&last_write[i]);
}
}
/* Scheduler pseudoinstruction lowerings to enable instruction pairings.
* Currently only support CUBEFACE -> *CUBEFACE1/+CUBEFACE2
*/
static bi_instr *
bi_lower_cubeface(bi_context *ctx,
struct bi_clause_state *clause, struct bi_tuple_state *tuple)
{
bi_instr *pinstr = tuple->add;
bi_builder b = bi_init_builder(ctx, bi_before_instr(pinstr));
bi_instr *cubeface1 = bi_cubeface1_to(&b, pinstr->dest[0],
pinstr->src[0], pinstr->src[1], pinstr->src[2]);
pinstr->op = BI_OPCODE_CUBEFACE2;
pinstr->dest[0] = pinstr->dest[1];
bi_drop_dests(pinstr, 1);
pinstr->src[0] = cubeface1->dest[0];
bi_drop_srcs(pinstr, 1);
return cubeface1;
}
/* Psuedo arguments are (rbase, address lo, address hi). We need *ATOM_C.i32 to
* have the arguments (address lo, address hi, rbase), and +ATOM_CX to have the
* arguments (rbase, address lo, address hi, rbase) */
static bi_instr *
bi_lower_atom_c(bi_context *ctx, struct bi_clause_state *clause, struct
bi_tuple_state *tuple)
{
bi_instr *pinstr = tuple->add;
bi_builder b = bi_init_builder(ctx, bi_before_instr(pinstr));
bi_instr *atom_c = bi_atom_c_return_i32(&b,
pinstr->src[1], pinstr->src[2], pinstr->src[0],
pinstr->atom_opc);
if (bi_is_null(pinstr->dest[0]))
atom_c->op = BI_OPCODE_ATOM_C_I32;
bi_instr *atom_cx = bi_atom_cx_to(&b, pinstr->dest[0], pinstr->src[0],
pinstr->src[1], pinstr->src[2], pinstr->src[0],
pinstr->sr_count);
tuple->add = atom_cx;
bi_remove_instruction(pinstr);
return atom_c;
}
static bi_instr *
bi_lower_atom_c1(bi_context *ctx, struct bi_clause_state *clause, struct
bi_tuple_state *tuple)
{
bi_instr *pinstr = tuple->add;
bi_builder b = bi_init_builder(ctx, bi_before_instr(pinstr));
bi_instr *atom_c = bi_atom_c1_return_i32(&b,
pinstr->src[0], pinstr->src[1], pinstr->atom_opc);
if (bi_is_null(pinstr->dest[0]))
atom_c->op = BI_OPCODE_ATOM_C1_I32;
bi_instr *atom_cx = bi_atom_cx_to(&b, pinstr->dest[0], bi_null(),
pinstr->src[0], pinstr->src[1], bi_dontcare(&b),
pinstr->sr_count);
tuple->add = atom_cx;
bi_remove_instruction(pinstr);
return atom_c;
}
static bi_instr *
bi_lower_seg_add(bi_context *ctx,
struct bi_clause_state *clause, struct bi_tuple_state *tuple)
{
bi_instr *pinstr = tuple->add;
bi_builder b = bi_init_builder(ctx, bi_before_instr(pinstr));
bi_instr *fma = bi_seg_add_to(&b, pinstr->dest[0], pinstr->src[0],
pinstr->preserve_null, pinstr->seg);
pinstr->op = BI_OPCODE_SEG_ADD;
pinstr->src[0] = pinstr->src[1];
bi_drop_srcs(pinstr, 1);
assert(pinstr->dest[0].type == BI_INDEX_REGISTER);
pinstr->dest[0].value += 1;
return fma;
}
static bi_instr *
bi_lower_dtsel(bi_context *ctx,
struct bi_clause_state *clause, struct bi_tuple_state *tuple)
{
bi_instr *add = tuple->add;
bi_builder b = bi_init_builder(ctx, bi_before_instr(add));
bi_instr *dtsel = bi_dtsel_imm_to(&b, bi_temp(b.shader),
add->src[0], add->table);
assert(add->nr_srcs >= 1);
add->src[0] = dtsel->dest[0];
assert(bi_supports_dtsel(add));
return dtsel;
}
/* Flatten linked list to array for O(1) indexing */
static bi_instr **
bi_flatten_block(bi_block *block, unsigned *len)
{
if (list_is_empty(&block->instructions))
return NULL;
*len = list_length(&block->instructions);
bi_instr **instructions = malloc(sizeof(bi_instr *) * (*len));
unsigned i = 0;
bi_foreach_instr_in_block(block, ins)
instructions[i++] = ins;
return instructions;
}
/* The worklist would track instructions without outstanding dependencies. For
* debug, force in-order scheduling (no dependency graph is constructed).
*/
static struct bi_worklist
bi_initialize_worklist(bi_block *block, bool inorder, bool is_blend)
{
struct bi_worklist st = { };
st.instructions = bi_flatten_block(block, &st.count);
if (!st.count)
return st;
st.dependents = calloc(st.count, sizeof(st.dependents[0]));
st.dep_counts = calloc(st.count, sizeof(st.dep_counts[0]));
bi_create_dependency_graph(st, inorder, is_blend);
st.worklist = calloc(BITSET_WORDS(st.count), sizeof(BITSET_WORD));
for (unsigned i = 0; i < st.count; ++i) {
if (st.dep_counts[i] == 0)
BITSET_SET(st.worklist, i);
}
return st;
}
static void
bi_free_worklist(struct bi_worklist st)
{
free(st.dep_counts);
free(st.dependents);
free(st.instructions);
free(st.worklist);
}
static void
bi_update_worklist(struct bi_worklist st, unsigned idx)
{
assert(st.dep_counts[idx] == 0);
if (!st.dependents[idx])
return;
/* Iterate each dependent to remove one dependency (`done`),
* adding dependents to the worklist where possible. */
unsigned i;
BITSET_FOREACH_SET(i, st.dependents[idx], st.count) {
assert(st.dep_counts[i] != 0);
unsigned new_deps = --st.dep_counts[i];
if (new_deps == 0)
BITSET_SET(st.worklist, i);
}
free(st.dependents[idx]);
}
/* Scheduler predicates */
/* IADDC.i32 can implement IADD.u32 if no saturation or swizzling is in use */
static bool
bi_can_iaddc(bi_instr *ins)
{
return (ins->op == BI_OPCODE_IADD_U32 && !ins->saturate &&
ins->src[0].swizzle == BI_SWIZZLE_H01 &&
ins->src[1].swizzle == BI_SWIZZLE_H01);
}
/*
* The encoding of *FADD.v2f16 only specifies a single abs flag. All abs
* encodings are permitted by swapping operands; however, this scheme fails if
* both operands are equal. Test for this case.
*/
static bool
bi_impacted_abs(bi_instr *I)
{
return I->src[0].abs && I->src[1].abs &&
bi_is_word_equiv(I->src[0], I->src[1]);
}
bool
bi_can_fma(bi_instr *ins)
{
/* +IADD.i32 -> *IADDC.i32 */
if (bi_can_iaddc(ins))
return true;
/* +MUX -> *CSEL */
if (bi_can_replace_with_csel(ins))
return true;
/* *FADD.v2f16 has restricted abs modifiers, use +FADD.v2f16 instead */
if (ins->op == BI_OPCODE_FADD_V2F16 && bi_impacted_abs(ins))
return false;
/* TODO: some additional fp16 constraints */
return bi_opcode_props[ins->op].fma;
}
static bool
bi_impacted_fadd_widens(bi_instr *I)
{
enum bi_swizzle swz0 = I->src[0].swizzle;
enum bi_swizzle swz1 = I->src[1].swizzle;
return (swz0 == BI_SWIZZLE_H00 && swz1 == BI_SWIZZLE_H11) ||
(swz0 == BI_SWIZZLE_H11 && swz1 == BI_SWIZZLE_H11) ||
(swz0 == BI_SWIZZLE_H11 && swz1 == BI_SWIZZLE_H00);
}
bool
bi_can_add(bi_instr *ins)
{
/* +FADD.v2f16 lacks clamp modifier, use *FADD.v2f16 instead */
if (ins->op == BI_OPCODE_FADD_V2F16 && ins->clamp)
return false;
/* +FCMP.v2f16 lacks abs modifier, use *FCMP.v2f16 instead */
if (ins->op == BI_OPCODE_FCMP_V2F16 && (ins->src[0].abs || ins->src[1].abs))
return false;
/* +FADD.f32 has restricted widens, use +FADD.f32 for the full set */
if (ins->op == BI_OPCODE_FADD_F32 && bi_impacted_fadd_widens(ins))
return false;
/* TODO: some additional fp16 constraints */
return bi_opcode_props[ins->op].add;
}
/* Architecturally, no single instruction has a "not last" constraint. However,
* pseudoinstructions writing multiple destinations (expanding to multiple
* paired instructions) can run afoul of the "no two writes on the last clause"
* constraint, so we check for that here.
*
* Exception to the exception: TEXC_DUAL, which writes to multiple sets of
* staging registers. Staging registers bypass the usual register write
* mechanism so this restriction does not apply.
*/
static bool
bi_must_not_last(bi_instr *ins)
{
return (ins->nr_dests >= 2) && (ins->op != BI_OPCODE_TEXC_DUAL);
}
/* Check for a message-passing instruction. +DISCARD.f32 is special-cased; we
* treat it as a message-passing instruction for the purpose of scheduling
* despite no passing no logical message. Otherwise invalid encoding faults may
* be raised for unknown reasons (possibly an errata).
*/
bool
bi_must_message(bi_instr *ins)
{
return (bi_opcode_props[ins->op].message != BIFROST_MESSAGE_NONE) ||
(ins->op == BI_OPCODE_DISCARD_F32);
}
static bool
bi_fma_atomic(enum bi_opcode op)
{
switch (op) {
case BI_OPCODE_ATOM_C_I32:
case BI_OPCODE_ATOM_C_I64:
case BI_OPCODE_ATOM_C1_I32:
case BI_OPCODE_ATOM_C1_I64:
case BI_OPCODE_ATOM_C1_RETURN_I32:
case BI_OPCODE_ATOM_C1_RETURN_I64:
case BI_OPCODE_ATOM_C_RETURN_I32:
case BI_OPCODE_ATOM_C_RETURN_I64:
case BI_OPCODE_ATOM_POST_I32:
case BI_OPCODE_ATOM_POST_I64:
case BI_OPCODE_ATOM_PRE_I64:
return true;
default:
return false;
}
}
bool
bi_reads_zero(bi_instr *ins)
{
return !(bi_fma_atomic(ins->op) || ins->op == BI_OPCODE_IMULD);
}
bool
bi_reads_temps(bi_instr *ins, unsigned src)
{
switch (ins->op) {
/* Cannot permute a temporary */
case BI_OPCODE_CLPER_I32:
case BI_OPCODE_CLPER_OLD_I32:
return src != 0;
/* ATEST isn't supposed to be restricted, but in practice it always
* wants to source its coverage mask input (source 0) from register 60,
* which won't work properly if we put the input in a temp. This
* requires workarounds in both RA and clause scheduling.
*/
case BI_OPCODE_ATEST:
return src != 0;
case BI_OPCODE_IMULD:
return false;
default:
return true;
}
}
static bool
bi_impacted_t_modifiers(bi_instr *I, unsigned src)
{
assert(src < I->nr_srcs);
enum bi_swizzle swizzle = I->src[src].swizzle;
switch (I->op) {
case BI_OPCODE_F16_TO_F32:
case BI_OPCODE_F16_TO_S32:
case BI_OPCODE_F16_TO_U32:
case BI_OPCODE_MKVEC_V2I16:
case BI_OPCODE_S16_TO_F32:
case BI_OPCODE_S16_TO_S32:
case BI_OPCODE_U16_TO_F32:
case BI_OPCODE_U16_TO_U32:
return (swizzle != BI_SWIZZLE_H00);
case BI_OPCODE_BRANCH_F32:
case BI_OPCODE_LOGB_F32:
case BI_OPCODE_ILOGB_F32:
case BI_OPCODE_FADD_F32:
case BI_OPCODE_FCMP_F32:
case BI_OPCODE_FREXPE_F32:
case BI_OPCODE_FREXPM_F32:
case BI_OPCODE_FROUND_F32:
return (swizzle != BI_SWIZZLE_H01);
case BI_OPCODE_IADD_S32:
case BI_OPCODE_IADD_U32:
case BI_OPCODE_ISUB_S32:
case BI_OPCODE_ISUB_U32:
case BI_OPCODE_IADD_V4S8:
case BI_OPCODE_IADD_V4U8:
case BI_OPCODE_ISUB_V4S8:
case BI_OPCODE_ISUB_V4U8:
return (src == 1) && (swizzle != BI_SWIZZLE_H01);
case BI_OPCODE_S8_TO_F32:
case BI_OPCODE_S8_TO_S32:
case BI_OPCODE_U8_TO_F32:
case BI_OPCODE_U8_TO_U32:
return (swizzle != BI_SWIZZLE_B0000);
case BI_OPCODE_V2S8_TO_V2F16:
case BI_OPCODE_V2S8_TO_V2S16:
case BI_OPCODE_V2U8_TO_V2F16:
case BI_OPCODE_V2U8_TO_V2U16:
return (swizzle != BI_SWIZZLE_B0022);
case BI_OPCODE_IADD_V2S16:
case BI_OPCODE_IADD_V2U16:
case BI_OPCODE_ISUB_V2S16:
case BI_OPCODE_ISUB_V2U16:
return (src == 1) && (swizzle >= BI_SWIZZLE_H11);
#if 0
/* Restriction on IADD in 64-bit clauses on G72 */
case BI_OPCODE_IADD_S64:
case BI_OPCODE_IADD_U64:
return (src == 1) && (swizzle != BI_SWIZZLE_D0);
#endif
default:
return false;
}
}
bool
bi_reads_t(bi_instr *ins, unsigned src)
{
/* Branch offset cannot come from passthrough */
if (bi_opcode_props[ins->op].branch)
return src != 2;
/* Table can never read passthrough */
if (bi_opcode_props[ins->op].table)
return false;
/* Staging register reads may happen before the succeeding register
* block encodes a write, so effectively there is no passthrough */
if (bi_is_staging_src(ins, src))
return false;
/* Bifrost cores newer than Mali G71 have restrictions on swizzles on
* same-cycle temporaries. Check the list for these hazards. */
if (bi_impacted_t_modifiers(ins, src))
return false;
/* Descriptor must not come from a passthrough */
switch (ins->op) {
case BI_OPCODE_LD_CVT:
case BI_OPCODE_LD_TILE:
case BI_OPCODE_ST_CVT:
case BI_OPCODE_ST_TILE:
case BI_OPCODE_TEXC:
case BI_OPCODE_TEXC_DUAL:
return src != 2;
case BI_OPCODE_BLEND:
return src != 2 && src != 3;
/* +JUMP can't read the offset from T */
case BI_OPCODE_JUMP:
return false;
/* Else, just check if we can read any temps */
default:
return bi_reads_temps(ins, src);
}
}
/* Counts the number of 64-bit constants required by a clause. TODO: We
* might want to account for merging, right now we overestimate, but
* that's probably fine most of the time */
static unsigned
bi_nconstants(struct bi_clause_state *clause)
{
unsigned count_32 = 0;
for (unsigned i = 0; i < ARRAY_SIZE(clause->consts); ++i)
count_32 += clause->consts[i].constant_count;
return DIV_ROUND_UP(count_32, 2);
}
/* Would there be space for constants if we added one tuple? */
static bool
bi_space_for_more_constants(struct bi_clause_state *clause)
{
return (bi_nconstants(clause) < 13 - (clause->tuple_count + 1));
}
/* Updates the FAU assignment for a tuple. A valid FAU assignment must be
* possible (as a precondition), though not necessarily on the selected unit;
* this is gauranteed per-instruction by bi_lower_fau and per-tuple by
* bi_instr_schedulable */
static bool
bi_update_fau(struct bi_clause_state *clause,
struct bi_tuple_state *tuple,
bi_instr *instr, bool fma, bool destructive)
{
/* Maintain our own constants, for nondestructive mode */
uint32_t copied_constants[2], copied_count;
unsigned *constant_count = &tuple->constant_count;
uint32_t *constants = tuple->constants;
enum bir_fau fau = tuple->fau;
if (!destructive) {
memcpy(copied_constants, tuple->constants,
(*constant_count) * sizeof(constants[0]));
copied_count = tuple->constant_count;
constant_count = &copied_count;
constants = copied_constants;
}
bi_foreach_src(instr, s) {
bi_index src = instr->src[s];
if (src.type == BI_INDEX_FAU) {
bool no_constants = *constant_count == 0;
bool no_other_fau = (fau == src.value) || !fau;
bool mergable = no_constants && no_other_fau;
if (destructive) {
assert(mergable);
tuple->fau = src.value;
} else if (!mergable) {
return false;
}
fau = src.value;
} else if (src.type == BI_INDEX_CONSTANT) {
/* No need to reserve space if we have a fast 0 */
if (src.value == 0 && fma && bi_reads_zero(instr))
continue;
/* If there is a branch target, #0 by convention is the
* PC-relative offset to the target */
bool pcrel = instr->branch_target && src.value == 0;
bool found = false;
for (unsigned i = 0; i < *constant_count; ++i) {
found |= (constants[i] == src.value) &&
(i != tuple->pcrel_idx);
}
/* pcrel constants are unique, so don't match */
if (found && !pcrel)
continue;
bool no_fau = (*constant_count > 0) || !fau;
bool mergable = no_fau && ((*constant_count) < 2);
if (destructive) {
assert(mergable);
if (pcrel)
tuple->pcrel_idx = *constant_count;
} else if (!mergable)
return false;
constants[(*constant_count)++] = src.value;
}
}
/* Constants per clause may be limited by tuple count */
bool room_for_constants = (*constant_count == 0) ||
bi_space_for_more_constants(clause);
if (destructive)
assert(room_for_constants);
else if (!room_for_constants)
return false;
return true;
}
/* Given an in-progress tuple, a candidate new instruction to add to the tuple,
* and a source (index) from that candidate, determine whether this source is
* "new", in the sense of requiring an additional read slot. That is, checks
* whether the specified source reads from the register file via a read slot
* (determined by its type and placement) and whether the source was already
* specified by a prior read slot (to avoid double counting) */
static bool
bi_tuple_is_new_src(bi_instr *instr, struct bi_reg_state *reg, unsigned src_idx)
{
assert(src_idx < instr->nr_srcs);
bi_index src = instr->src[src_idx];
/* Only consider sources which come from the register file */
if (!(src.type == BI_INDEX_NORMAL || src.type == BI_INDEX_REGISTER))
return false;
/* Staging register reads bypass the usual register file mechanism */
if (bi_is_staging_src(instr, src_idx))
return false;
/* If a source is already read in the tuple, it is already counted */
for (unsigned t = 0; t < reg->nr_reads; ++t)
if (bi_is_word_equiv(src, reg->reads[t]))
return false;
/* If a source is read in _this instruction_, it is already counted */
for (unsigned t = 0; t < src_idx; ++t)
if (bi_is_word_equiv(src, instr->src[t]))
return false;
return true;
}
/* Given two tuples in source order, count the number of register reads of the
* successor, determined as the number of unique words accessed that aren't
* written by the predecessor (since those are tempable).
*/
static unsigned
bi_count_succ_reads(bi_index t0, bi_index t1,
bi_index *succ_reads, unsigned nr_succ_reads)
{
unsigned reads = 0;
for (unsigned i = 0; i < nr_succ_reads; ++i) {
bool unique = true;
for (unsigned j = 0; j < i; ++j)
if (bi_is_word_equiv(succ_reads[i], succ_reads[j]))
unique = false;
if (!unique)
continue;
if (bi_is_word_equiv(succ_reads[i], t0))
continue;
if (bi_is_word_equiv(succ_reads[i], t1))
continue;
reads++;
}
return reads;
}
/* Not all instructions can read from the staging passthrough (as determined by
* reads_t), check if a given pair of instructions has such a restriction. Note
* we also use this mechanism to prevent data races around staging register
* reads, so we allow the input source to potentially be vector-valued */
static bool
bi_has_staging_passthrough_hazard(bi_index fma, bi_instr *add)
{
bi_foreach_src(add, s) {
bi_index src = add->src[s];
if (src.type != BI_INDEX_REGISTER)
continue;
unsigned count = bi_count_read_registers(add, s);
bool read = false;
for (unsigned d = 0; d < count; ++d)
read |= bi_is_equiv(fma, bi_register(src.value + d));
if (read && !bi_reads_t(add, s))
return true;
}
return false;
}
/* Likewise for cross-tuple passthrough (reads_temps) */
static bool
bi_has_cross_passthrough_hazard(bi_tuple *succ, bi_instr *ins)
{
if (ins->nr_dests == 0)
return false;
bi_foreach_instr_in_tuple(succ, pins) {
bi_foreach_src(pins, s) {
if (bi_is_word_equiv(ins->dest[0], pins->src[s]) &&
!bi_reads_temps(pins, s))
return true;
}
}
return false;
}
/* Is a register written other than the staging mechanism? ATEST is special,
* writing to both a staging register and a regular register (fixed packing).
* BLEND is special since it has to write r48 the normal way even if it never
* gets read. This depends on liveness analysis, as a register is not needed
* for a write that will be discarded after one tuple. */
static unsigned
bi_write_count(bi_instr *instr, uint64_t live_after_temp)
{
if (instr->op == BI_OPCODE_ATEST || instr->op == BI_OPCODE_BLEND)
return 1;
unsigned count = 0;
bi_foreach_dest(instr, d) {
if (d == 0 && bi_opcode_props[instr->op].sr_write)
continue;
assert(instr->dest[0].type == BI_INDEX_REGISTER);
if (live_after_temp & BITFIELD64_BIT(instr->dest[0].value))
count++;
}
return count;
}
/*
* Test if an instruction required flush-to-zero mode. Currently only supported
* for f16<-->f32 conversions to implement fquantize16
*/
static bool
bi_needs_ftz(bi_instr *I)
{
return (I->op == BI_OPCODE_F16_TO_F32 ||
I->op == BI_OPCODE_V2F32_TO_V2F16) && I->ftz;
}
/*
* Test if an instruction would be numerically incompatible with the clause. At
* present we only consider flush-to-zero modes.
*/
static bool
bi_numerically_incompatible(struct bi_clause_state *clause, bi_instr *instr)
{
return (clause->ftz != BI_FTZ_STATE_NONE) &&
((clause->ftz == BI_FTZ_STATE_ENABLE) != bi_needs_ftz(instr));
}
/* Instruction placement entails two questions: what subset of instructions in
* the block can legally be scheduled? and of those which is the best? That is,
* we seek to maximize a cost function on a subset of the worklist satisfying a
* particular predicate. The necessary predicate is determined entirely by
* Bifrost's architectural limitations and is described in the accompanying
* whitepaper. The cost function is a heuristic. */
static bool
bi_instr_schedulable(bi_instr *instr,
struct bi_clause_state *clause,
struct bi_tuple_state *tuple,
uint64_t live_after_temp,
bool fma)
{
/* The units must match */
if ((fma && !bi_can_fma(instr)) || (!fma && !bi_can_add(instr)))
return false;
/* There can only be one message-passing instruction per clause */
if (bi_must_message(instr) && clause->message)
return false;
/* Some instructions have placement requirements */
if (bi_opcode_props[instr->op].last && !tuple->last)
return false;
if (bi_must_not_last(instr) && tuple->last)
return false;
/* Numerical properties must be compatible with the clause */
if (bi_numerically_incompatible(clause, instr))
return false;
/* Message-passing instructions are not guaranteed write within the
* same clause (most likely they will not), so if a later instruction
* in the clause accesses the destination, the message-passing
* instruction can't be scheduled */
if (bi_opcode_props[instr->op].sr_write) {
bi_foreach_dest(instr, d) {
unsigned nr = bi_count_write_registers(instr, d);
assert(instr->dest[d].type == BI_INDEX_REGISTER);
unsigned reg = instr->dest[d].value;
for (unsigned i = 0; i < clause->access_count; ++i) {
bi_index idx = clause->accesses[i];
for (unsigned d = 0; d < nr; ++d) {
if (bi_is_equiv(bi_register(reg + d), idx))
return false;
}
}
}
}
if (bi_opcode_props[instr->op].sr_read && !bi_is_null(instr->src[0])) {
unsigned nr = bi_count_read_registers(instr, 0);
assert(instr->src[0].type == BI_INDEX_REGISTER);
unsigned reg = instr->src[0].value;
for (unsigned i = 0; i < clause->access_count; ++i) {
bi_index idx = clause->accesses[i];
for (unsigned d = 0; d < nr; ++d) {
if (bi_is_equiv(bi_register(reg + d), idx))
return false;
}
}
}
/* If FAU is already assigned, we may not disrupt that. Do a
* non-disruptive test update */
if (!bi_update_fau(clause, tuple, instr, fma, false))
return false;
/* If this choice of FMA would force a staging passthrough, the ADD
* instruction must support such a passthrough */
if (tuple->add && instr->nr_dests && bi_has_staging_passthrough_hazard(instr->dest[0], tuple->add))
return false;
/* If this choice of destination would force a cross-tuple passthrough, the next tuple must support that */
if (tuple->prev && bi_has_cross_passthrough_hazard(tuple->prev, instr))
return false;
/* Register file writes are limited */
unsigned total_writes = tuple->reg.nr_writes;
total_writes += bi_write_count(instr, live_after_temp);
/* Last tuple in a clause can only write a single value */
if (tuple->last && total_writes > 1)
return false;
/* Register file reads are limited, so count unique */
unsigned unique_new_srcs = 0;
bi_foreach_src(instr, s) {
if (bi_tuple_is_new_src(instr, &tuple->reg, s))
unique_new_srcs++;
}
unsigned total_srcs = tuple->reg.nr_reads + unique_new_srcs;
bool can_spill_to_moves = (!tuple->add);
can_spill_to_moves &= (bi_nconstants(clause) < 13 - (clause->tuple_count + 2));
can_spill_to_moves &= (clause->tuple_count < 7);
/* However, we can get an extra 1 or 2 sources by inserting moves */
if (total_srcs > (can_spill_to_moves ? 4 : 3))
return false;
/* Count effective reads for the successor */
unsigned succ_reads = 0;
if (instr->nr_dests) {
bool has_t1 = tuple->add && tuple->add->nr_dests;
succ_reads = bi_count_succ_reads(instr->dest[0],
has_t1 ? tuple->add->dest[0] : bi_null(),
tuple->prev_reads,
tuple->nr_prev_reads);
}
/* Successor must satisfy R+W <= 4, so we require W <= 4-R */
if ((signed) total_writes > (4 - (signed) succ_reads))
return false;
return true;
}
static signed
bi_instr_cost(bi_instr *instr, struct bi_tuple_state *tuple)
{
signed cost = 0;
/* Instructions that can schedule to either FMA or to ADD should be
* deprioritized since they're easier to reschedule elsewhere */
if (bi_can_fma(instr) && bi_can_add(instr))
cost++;
/* Message-passing instructions impose constraints on the registers
* later in the clause, so schedule them as late within a clause as
* possible (<==> prioritize them since we're backwards <==> decrease
* cost) */
if (bi_must_message(instr))
cost--;
/* Last instructions are big constraints (XXX: no effect on shader-db) */
if (bi_opcode_props[instr->op].last)
cost -= 2;
return cost;
}
static unsigned
bi_choose_index(struct bi_worklist st,
struct bi_clause_state *clause,
struct bi_tuple_state *tuple,
uint64_t live_after_temp,
bool fma)
{
unsigned i, best_idx = ~0;
signed best_cost = INT_MAX;
BITSET_FOREACH_SET(i, st.worklist, st.count) {
bi_instr *instr = st.instructions[i];
if (!bi_instr_schedulable(instr, clause, tuple, live_after_temp, fma))
continue;
signed cost = bi_instr_cost(instr, tuple);
/* Tie break in favour of later instructions, under the
* assumption this promotes temporary usage (reducing pressure
* on the register file). This is a side effect of a prepass
* scheduling for pressure. */
if (cost <= best_cost) {
best_idx = i;
best_cost = cost;
}
}
return best_idx;
}
static void
bi_pop_instr(struct bi_clause_state *clause, struct bi_tuple_state *tuple,
bi_instr *instr, uint64_t live_after_temp, bool fma)
{
bi_update_fau(clause, tuple, instr, fma, true);
assert(clause->access_count + instr->nr_srcs + instr->nr_dests <= ARRAY_SIZE(clause->accesses));
memcpy(clause->accesses + clause->access_count,
instr->src, sizeof(instr->src[0]) * instr->nr_srcs);
clause->access_count += instr->nr_srcs;
memcpy(clause->accesses + clause->access_count,
instr->dest, sizeof(instr->dest[0]) * instr->nr_dests);
clause->access_count += instr->nr_dests;
tuple->reg.nr_writes += bi_write_count(instr, live_after_temp);
bi_foreach_src(instr, s) {
if (bi_tuple_is_new_src(instr, &tuple->reg, s))
tuple->reg.reads[tuple->reg.nr_reads++] = instr->src[s];
}
/* This could be optimized to allow pairing integer instructions with
* special flush-to-zero instructions, but punting on this until we have
* a workload that cares.
*/
clause->ftz = bi_needs_ftz(instr) ? BI_FTZ_STATE_ENABLE :
BI_FTZ_STATE_DISABLE;
}
/* Choose the best instruction and pop it off the worklist. Returns NULL if no
* instruction is available. This function is destructive. */
static bi_instr *
bi_take_instr(bi_context *ctx, struct bi_worklist st,
struct bi_clause_state *clause,
struct bi_tuple_state *tuple,
uint64_t live_after_temp,
bool fma)
{
if (tuple->add && tuple->add->op == BI_OPCODE_CUBEFACE)
return bi_lower_cubeface(ctx, clause, tuple);
else if (tuple->add && tuple->add->op == BI_OPCODE_ATOM_RETURN_I32)
return bi_lower_atom_c(ctx, clause, tuple);
else if (tuple->add && tuple->add->op == BI_OPCODE_ATOM1_RETURN_I32)
return bi_lower_atom_c1(ctx, clause, tuple);
else if (tuple->add && tuple->add->op == BI_OPCODE_SEG_ADD_I64)
return bi_lower_seg_add(ctx, clause, tuple);
else if (tuple->add && tuple->add->table)
return bi_lower_dtsel(ctx, clause, tuple);
/* TODO: Optimize these moves */
if (!fma && tuple->nr_prev_reads > 3) {
/* Only spill by one source for now */
assert(tuple->nr_prev_reads == 4);
/* Pick a source to spill */
bi_index src = tuple->prev_reads[0];
/* Schedule the spill */
bi_builder b = bi_init_builder(ctx, bi_before_tuple(tuple->prev));
bi_instr *mov = bi_mov_i32_to(&b, src, src);
bi_pop_instr(clause, tuple, mov, live_after_temp, fma);
return mov;
}
#ifndef NDEBUG
/* Don't pair instructions if debugging */
if ((bifrost_debug & BIFROST_DBG_NOSCHED) && tuple->add)
return NULL;
#endif
unsigned idx = bi_choose_index(st, clause, tuple, live_after_temp, fma);
if (idx >= st.count)
return NULL;
/* Update state to reflect taking the instruction */
bi_instr *instr = st.instructions[idx];
BITSET_CLEAR(st.worklist, idx);
bi_update_worklist(st, idx);
bi_pop_instr(clause, tuple, instr, live_after_temp, fma);
/* Fixups */
bi_builder b = bi_init_builder(ctx, bi_before_instr(instr));
if (instr->op == BI_OPCODE_IADD_U32 && fma) {
assert(bi_can_iaddc(instr));
bi_instr *iaddc =
bi_iaddc_i32_to(&b, instr->dest[0], instr->src[0],
instr->src[1], bi_zero());
bi_remove_instruction(instr);
instr = iaddc;
} else if (fma && bi_can_replace_with_csel(instr)) {
bi_instr *csel = bi_csel_from_mux(&b, instr, false);
bi_remove_instruction(instr);
instr = csel;
}
return instr;
}
/* Variant of bi_rewrite_index_src_single that uses word-equivalence, rewriting
* to a passthrough register. If except_sr is true, the staging sources are
* skipped, so staging register reads are not accidentally encoded as
* passthrough (which is impossible) */
static void
bi_use_passthrough(bi_instr *ins, bi_index old,
enum bifrost_packed_src new,
bool except_sr)
{
/* Optional for convenience */
if (!ins)
return;
assert(!bi_is_null(old));
bi_foreach_src(ins, i) {
if ((i == 0 || i == 4) && except_sr)
continue;
if (bi_is_word_equiv(ins->src[i], old)) {
ins->src[i].type = BI_INDEX_PASS;
ins->src[i].value = new;
ins->src[i].offset = 0;
}
}
}
/* Rewrites an adjacent pair of tuples _prec_eding and _succ_eding to use
* intertuple passthroughs where necessary. Passthroughs are allowed as a
* post-condition of scheduling. Note we rewrite ADD first, FMA second --
* opposite the order of execution. This is deliberate -- if both FMA and ADD
* write to the same logical register, the next executed tuple will get the
* latter result. There's no interference issue under the assumption of correct
* register allocation. */
static void
bi_rewrite_passthrough(bi_tuple prec, bi_tuple succ)
{
bool sr_read = succ.add ? bi_opcode_props[succ.add->op].sr_read : false;
if (prec.add && prec.add->nr_dests) {
bi_use_passthrough(succ.fma, prec.add->dest[0], BIFROST_SRC_PASS_ADD, false);
bi_use_passthrough(succ.add, prec.add->dest[0], BIFROST_SRC_PASS_ADD, sr_read);
}
if (prec.fma && prec.fma->nr_dests) {
bi_use_passthrough(succ.fma, prec.fma->dest[0], BIFROST_SRC_PASS_FMA, false);
bi_use_passthrough(succ.add, prec.fma->dest[0], BIFROST_SRC_PASS_FMA, sr_read);
}
}
static void
bi_rewrite_fau_to_pass(bi_tuple *tuple)
{
bi_foreach_instr_and_src_in_tuple(tuple, ins, s) {
if (ins->src[s].type != BI_INDEX_FAU) continue;
bi_index pass = bi_passthrough(ins->src[s].offset ?
BIFROST_SRC_FAU_HI : BIFROST_SRC_FAU_LO);
bi_replace_src(ins, s, pass);
}
}
static void
bi_rewrite_zero(bi_instr *ins, bool fma)
{
bi_index zero = bi_passthrough(fma ? BIFROST_SRC_STAGE : BIFROST_SRC_FAU_LO);
bi_foreach_src(ins, s) {
bi_index src = ins->src[s];
if (src.type == BI_INDEX_CONSTANT && src.value == 0)
bi_replace_src(ins, s, zero);
}
}
/* Assumes #0 to {T, FAU} rewrite has already occurred */
static void
bi_rewrite_constants_to_pass(bi_tuple *tuple, uint64_t constant, bool pcrel)
{
bi_foreach_instr_and_src_in_tuple(tuple, ins, s) {
if (ins->src[s].type != BI_INDEX_CONSTANT) continue;
uint32_t cons = ins->src[s].value;
ASSERTED bool lo = (cons == (constant & 0xffffffff));
bool hi = (cons == (constant >> 32ull));
/* PC offsets always live in the upper half, set to zero by
* convention before pack time. (This is safe, since if you
* wanted to compare against zero, you would use a BRANCHZ
* instruction instead.) */
if (cons == 0 && ins->branch_target != NULL) {
assert(pcrel);
hi = true;
lo = false;
} else if (pcrel) {
hi = false;
}
assert(lo || hi);
bi_replace_src(ins, s,
bi_passthrough(hi ? BIFROST_SRC_FAU_HI : BIFROST_SRC_FAU_LO));
}
}
/* Constructs a constant state given a tuple state. This has the
* postcondition that pcrel applies to the first constant by convention,
* and PC-relative constants will be #0 by convention here, so swap to
* match if needed */
static struct bi_const_state
bi_get_const_state(struct bi_tuple_state *tuple)
{
struct bi_const_state consts = {
.constant_count = tuple->constant_count,
.constants[0] = tuple->constants[0],
.constants[1] = tuple->constants[1],
.pcrel = tuple->add && tuple->add->branch_target,
};
/* pcrel applies to the first constant by convention, and
* PC-relative constants will be #0 by convention here, so swap
* to match if needed */
if (consts.pcrel && consts.constants[0]) {
assert(consts.constant_count == 2);
assert(consts.constants[1] == 0);
consts.constants[1] = consts.constants[0];
consts.constants[0] = 0;
}
return consts;
}
/* Merges constants in a clause, satisfying the following rules, assuming no
* more than one tuple has pcrel:
*
* 1. If a tuple has two constants, they must be packed together. If one is
* pcrel, it must be the high constant to use the M1=4 modification [sx64(E0) +
* (PC << 32)]. Otherwise choose an arbitrary order.
*
* 4. If a tuple has one constant, it may be shared with an existing
* pair that already contains that constant, or it may be combined with another
* (distinct) tuple of a single constant.
*
* This gaurantees a packing is possible. The next routine handles modification
* related swapping, to satisfy format 12 and the lack of modification for
* tuple count 5/8 in EC0.
*/
static uint64_t
bi_merge_u32(uint32_t c0, uint32_t c1, bool pcrel)
{
/* At this point in the constant merge algorithm, pcrel constants are
* treated as zero, so pcrel implies at least one constants is zero */
assert(!pcrel || (c0 == 0 || c1 == 0));
/* Order: pcrel, maximum non-pcrel, minimum non-pcrel */
uint32_t hi = pcrel ? 0 : MAX2(c0, c1);
uint32_t lo = (c0 == hi) ? c1 : c0;
/* Merge in the selected order */
return lo | (((uint64_t) hi) << 32ull);
}
static unsigned
bi_merge_pairs(struct bi_const_state *consts, unsigned tuple_count,
uint64_t *merged, unsigned *pcrel_pair)
{
unsigned merge_count = 0;
for (unsigned t = 0; t < tuple_count; ++t) {
if (consts[t].constant_count != 2) continue;
unsigned idx = ~0;
uint64_t val = bi_merge_u32(consts[t].constants[0],
consts[t].constants[1], consts[t].pcrel);
/* Skip the pcrel pair if assigned, because if one is assigned,
* this one is not pcrel by uniqueness so it's a mismatch */
for (unsigned s = 0; s < merge_count; ++s) {
if (merged[s] == val && (*pcrel_pair) != s) {
idx = s;
break;
}
}
if (idx == ~0) {
idx = merge_count++;
merged[idx] = val;
if (consts[t].pcrel)
(*pcrel_pair) = idx;
}
consts[t].word_idx = idx;
}
return merge_count;
}
static unsigned
bi_merge_singles(struct bi_const_state *consts, unsigned tuple_count,
uint64_t *pairs, unsigned pair_count, unsigned *pcrel_pair)
{
bool pending = false, pending_pcrel = false;
uint32_t pending_single = 0;
for (unsigned t = 0; t < tuple_count; ++t) {
if (consts[t].constant_count != 1) continue;
uint32_t val = consts[t].constants[0];
unsigned idx = ~0;
/* Try to match, but don't match pcrel with non-pcrel, even
* though we can merge a pcrel with a non-pcrel single */
for (unsigned i = 0; i < pair_count; ++i) {
bool lo = ((pairs[i] & 0xffffffff) == val);
bool hi = ((pairs[i] >> 32) == val);
bool match = (lo || hi);
match &= ((*pcrel_pair) != i);
if (match && !consts[t].pcrel) {
idx = i;
break;
}
}
if (idx == ~0) {
idx = pair_count;
if (pending && pending_single != val) {
assert(!(pending_pcrel && consts[t].pcrel));
bool pcrel = pending_pcrel || consts[t].pcrel;
if (pcrel)
*pcrel_pair = idx;
pairs[pair_count++] = bi_merge_u32(pending_single, val, pcrel);
pending = pending_pcrel = false;
} else {
pending = true;
pending_pcrel = consts[t].pcrel;
pending_single = val;
}
}
consts[t].word_idx = idx;
}
/* Shift so it works whether pending_pcrel is set or not */
if (pending) {
if (pending_pcrel)
*pcrel_pair = pair_count;
pairs[pair_count++] = ((uint64_t) pending_single) << 32ull;
}
return pair_count;
}
static unsigned
bi_merge_constants(struct bi_const_state *consts, uint64_t *pairs, unsigned *pcrel_idx)
{
unsigned pair_count = bi_merge_pairs(consts, 8, pairs, pcrel_idx);
return bi_merge_singles(consts, 8, pairs, pair_count, pcrel_idx);
}
/* Swap two constants at word i and i+1 by swapping their actual positions and
* swapping all references so the meaning of the clause is preserved */
static void
bi_swap_constants(struct bi_const_state *consts, uint64_t *pairs, unsigned i)
{
uint64_t tmp_pair = pairs[i + 0];
pairs[i + 0] = pairs[i + 1];
pairs[i + 1] = tmp_pair;
for (unsigned t = 0; t < 8; ++t) {
if (consts[t].word_idx == i)
consts[t].word_idx = (i + 1);
else if (consts[t].word_idx == (i + 1))
consts[t].word_idx = i;
}
}
/* Given merged constants, one of which might be PC-relative, fix up the M
* values so the PC-relative constant (if it exists) has the M1=4 modification
* and other constants are used as-is (which might require swapping) */
static unsigned
bi_apply_constant_modifiers(struct bi_const_state *consts,
uint64_t *pairs, unsigned *pcrel_idx,
unsigned tuple_count, unsigned constant_count)
{
unsigned start = bi_ec0_packed(tuple_count) ? 1 : 0;
/* Clauses with these tuple counts lack an M field for the packed EC0,
* so EC0 cannot be PC-relative, which might require swapping (and
* possibly adding an unused constant) to fit */
if (*pcrel_idx == 0 && (tuple_count == 5 || tuple_count == 8)) {
constant_count = MAX2(constant_count, 2);
*pcrel_idx = 1;
bi_swap_constants(consts, pairs, 0);
}
/* EC0 might be packed free, after that constants are packed in pairs
* (with clause format 12), with M1 values computed from the pair */
for (unsigned i = start; i < constant_count; i += 2) {
bool swap = false;
bool last = (i + 1) == constant_count;
unsigned A1 = (pairs[i] >> 60);
unsigned B1 = (pairs[i + 1] >> 60);
if (*pcrel_idx == i || *pcrel_idx == (i + 1)) {
/* PC-relative constant must be E0, not E1 */
swap = (*pcrel_idx == (i + 1));
/* Set M1 = 4 by noting (A - B) mod 16 = 4 is
* equivalent to A = (B + 4) mod 16 and that we can
* control A */
unsigned B = swap ? A1 : B1;
unsigned A = (B + 4) & 0xF;
pairs[*pcrel_idx] |= ((uint64_t) A) << 60;
/* Swapped if swap set, identity if swap not set */
*pcrel_idx = i;
} else {
/* Compute M1 value if we don't swap */
unsigned M1 = (16 + A1 - B1) & 0xF;
/* For M1 = 0 or M1 >= 8, the constants are unchanged,
* we have 0 < (A1 - B1) % 16 < 8, which implies (B1 -
* A1) % 16 >= 8, so swapping will let them be used
* unchanged */
swap = (M1 != 0) && (M1 < 8);
/* However, we can't swap the last constant, so we
* force M1 = 0 instead for this case */
if (last && swap) {
pairs[i + 1] |= pairs[i] & (0xfull << 60);
swap = false;
}
}
if (swap) {
assert(!last);
bi_swap_constants(consts, pairs, i);
}
}
return constant_count;
}
/* Schedule a single clause. If no instructions remain, return NULL. */
static bi_clause *
bi_schedule_clause(bi_context *ctx, bi_block *block, struct bi_worklist st, uint64_t *live)
{
struct bi_clause_state clause_state = { 0 };
bi_clause *clause = rzalloc(ctx, bi_clause);
bi_tuple *tuple = NULL;
const unsigned max_tuples = ARRAY_SIZE(clause->tuples);
/* TODO: Decide flow control better */
clause->flow_control = BIFROST_FLOW_NBTB;
/* The last clause can only write one instruction, so initialize that */
struct bi_reg_state reg_state = {};
bi_index prev_reads[5] = { bi_null() };
unsigned nr_prev_reads = 0;
/* We need to track future liveness. The main *live set tracks what is
* live at the current point int he program we are scheduling, but to
* determine temp eligibility, we instead want what will be live after
* the next tuple in the program. If you scheduled forwards, you'd need
* a crystall ball for this. Luckily we schedule backwards, so we just
* delay updates to the live_after_temp by an extra tuple. */
uint64_t live_after_temp = *live;
uint64_t live_next_tuple = live_after_temp;
do {
struct bi_tuple_state tuple_state = {
.last = (clause->tuple_count == 0),
.reg = reg_state,
.nr_prev_reads = nr_prev_reads,
.prev = tuple,
.pcrel_idx = ~0,
};
assert(nr_prev_reads < ARRAY_SIZE(prev_reads));
memcpy(tuple_state.prev_reads, prev_reads, sizeof(prev_reads));
unsigned idx = max_tuples - clause->tuple_count - 1;
tuple = &clause->tuples[idx];
if (clause->message && bi_opcode_props[clause->message->op].sr_read && !bi_is_null(clause->message->src[0])) {
unsigned nr = bi_count_read_registers(clause->message, 0);
live_after_temp |= (BITFIELD64_MASK(nr) << clause->message->src[0].value);
}
/* Since we schedule backwards, we schedule ADD first */
tuple_state.add = bi_take_instr(ctx, st, &clause_state, &tuple_state, live_after_temp, false);
tuple->fma = bi_take_instr(ctx, st, &clause_state, &tuple_state, live_after_temp, true);
tuple->add = tuple_state.add;
/* Update liveness from the new instructions */
if (tuple->add)
*live = bi_postra_liveness_ins(*live, tuple->add);
if (tuple->fma)
*live = bi_postra_liveness_ins(*live, tuple->fma);
/* Rotate in the new per-tuple liveness */
live_after_temp = live_next_tuple;
live_next_tuple = *live;
/* We may have a message, but only one per clause */
if (tuple->add && bi_must_message(tuple->add)) {
assert(!clause_state.message);
clause_state.message = true;
clause->message_type =
bi_message_type_for_instr(tuple->add);
clause->message = tuple->add;
/* We don't need to set dependencies for blend shaders
* because the BLEND instruction in the fragment
* shader should have already done the wait */
if (!ctx->inputs->is_blend) {
switch (tuple->add->op) {
case BI_OPCODE_ATEST:
clause->dependencies |= (1 << BIFROST_SLOT_ELDEST_DEPTH);
break;
case BI_OPCODE_LD_TILE:
case BI_OPCODE_ST_TILE:
clause->dependencies |= (1 << BIFROST_SLOT_ELDEST_COLOUR);
break;
case BI_OPCODE_BLEND:
clause->dependencies |= (1 << BIFROST_SLOT_ELDEST_DEPTH);
clause->dependencies |= (1 << BIFROST_SLOT_ELDEST_COLOUR);
break;
default:
break;
}
}
}
clause_state.consts[idx] = bi_get_const_state(&tuple_state);
/* Before merging constants, eliminate zeroes, otherwise the
* merging will fight over the #0 that never gets read (and is
* never marked as read by update_fau) */
if (tuple->fma && bi_reads_zero(tuple->fma))
bi_rewrite_zero(tuple->fma, true);
/* Rewrite away FAU, constant write is deferred */
if (!tuple_state.constant_count) {
tuple->fau_idx = tuple_state.fau;
bi_rewrite_fau_to_pass(tuple);
}
/* Use passthrough register for cross-stage accesses. Since
* there are just FMA and ADD stages, that means we rewrite to
* passthrough the sources of the ADD that read from the
* destination of the FMA */
if (tuple->fma && tuple->fma->nr_dests) {
bi_use_passthrough(tuple->add, tuple->fma->dest[0],
BIFROST_SRC_STAGE, false);
}
/* Don't add an empty tuple, unless the worklist has nothing
* but a (pseudo)instruction failing to schedule due to a "not
* last instruction" constraint */
int some_instruction = __bitset_ffs(st.worklist, BITSET_WORDS(st.count));
bool not_last = (some_instruction > 0) &&
bi_must_not_last(st.instructions[some_instruction - 1]);
bool insert_empty = tuple_state.last && not_last;
if (!(tuple->fma || tuple->add || insert_empty))
break;
clause->tuple_count++;
/* Adding enough tuple might overflow constants */
if (!bi_space_for_more_constants(&clause_state))
break;
#ifndef NDEBUG
/* Don't schedule more than 1 tuple if debugging */
if ((bifrost_debug & BIFROST_DBG_NOSCHED) && !insert_empty)
break;
#endif
/* Link through the register state */
STATIC_ASSERT(sizeof(prev_reads) == sizeof(tuple_state.reg.reads));
memcpy(prev_reads, tuple_state.reg.reads, sizeof(prev_reads));
nr_prev_reads = tuple_state.reg.nr_reads;
clause_state.tuple_count++;
} while(clause->tuple_count < 8);
/* Don't schedule an empty clause */
if (!clause->tuple_count)
return NULL;
/* Before merging, rewrite away any tuples that read only zero */
for (unsigned i = max_tuples - clause->tuple_count; i < max_tuples; ++i) {
bi_tuple *tuple = &clause->tuples[i];
struct bi_const_state *st = &clause_state.consts[i];
if (st->constant_count == 0 || st->constants[0] || st->constants[1] || st->pcrel)
continue;
bi_foreach_instr_in_tuple(tuple, ins)
bi_rewrite_zero(ins, false);
/* Constant has been demoted to FAU, so don't pack it separately */
st->constant_count = 0;
/* Default */
assert(tuple->fau_idx == BIR_FAU_ZERO);
}
uint64_t constant_pairs[8] = { 0 };
unsigned pcrel_idx = ~0;
unsigned constant_words =
bi_merge_constants(clause_state.consts, constant_pairs, &pcrel_idx);
constant_words = bi_apply_constant_modifiers(clause_state.consts,
constant_pairs, &pcrel_idx, clause->tuple_count,
constant_words);
clause->pcrel_idx = pcrel_idx;
for (unsigned i = max_tuples - clause->tuple_count; i < max_tuples; ++i) {
bi_tuple *tuple = &clause->tuples[i];
/* If no constants, leave FAU as it is, possibly defaulting to 0 */
if (clause_state.consts[i].constant_count == 0)
continue;
/* FAU is already handled */
assert(!tuple->fau_idx);
unsigned word_idx = clause_state.consts[i].word_idx;
assert(word_idx <= 8);
/* We could try to merge regardless of bottom bits as well, but
* that's probably diminishing returns */
uint64_t pair = constant_pairs[word_idx];
unsigned lo = pair & 0xF;
tuple->fau_idx = bi_constant_field(word_idx) | lo;
bi_rewrite_constants_to_pass(tuple, pair, word_idx == pcrel_idx);
}
clause->constant_count = constant_words;
memcpy(clause->constants, constant_pairs, sizeof(constant_pairs));
/* Branches must be last, so this can be factored out */
bi_instr *last = clause->tuples[max_tuples - 1].add;
clause->next_clause_prefetch = !last || (last->op != BI_OPCODE_JUMP);
clause->block = block;
clause->ftz = (clause_state.ftz == BI_FTZ_STATE_ENABLE);
/* We emit in reverse and emitted to the back of the tuples array, so
* move it up front for easy indexing */
memmove(clause->tuples,
clause->tuples + (max_tuples - clause->tuple_count),
clause->tuple_count * sizeof(clause->tuples[0]));
/* Use passthrough register for cross-tuple accesses. Note this is
* after the memmove, so this is forwards. Skip the first tuple since
* there is nothing before it to passthrough */
for (unsigned t = 1; t < clause->tuple_count; ++t)
bi_rewrite_passthrough(clause->tuples[t - 1], clause->tuples[t]);
return clause;
}
static void
bi_schedule_block(bi_context *ctx, bi_block *block)
{
list_inithead(&block->clauses);
/* Copy list to dynamic array */
struct bi_worklist st = bi_initialize_worklist(block,
bifrost_debug & BIFROST_DBG_INORDER,
ctx->inputs->is_blend);
if (!st.count) {
bi_free_worklist(st);
return;
}
/* We need to track liveness during scheduling in order to determine whether we can use temporary (passthrough) registers */
uint64_t live = block->reg_live_out;
/* Schedule as many clauses as needed to fill the block */
bi_clause *u = NULL;
while((u = bi_schedule_clause(ctx, block, st, &live)))
list_add(&u->link, &block->clauses);
/* Back-to-back bit affects only the last clause of a block,
* the rest are implicitly true */
if (!list_is_empty(&block->clauses)) {
bi_clause *last_clause = list_last_entry(&block->clauses, bi_clause, link);
if (bi_reconverge_branches(block))
last_clause->flow_control = BIFROST_FLOW_NBTB_UNCONDITIONAL;
}
/* Reorder instructions to match the new schedule. First remove
* existing instructions and then recreate the list */
bi_foreach_instr_in_block_safe(block, ins) {
list_del(&ins->link);
}
bi_foreach_clause_in_block(block, clause) {
for (unsigned i = 0; i < clause->tuple_count; ++i) {
bi_foreach_instr_in_tuple(&clause->tuples[i], ins) {
list_addtail(&ins->link, &block->instructions);
}
}
}
block->scheduled = true;
#ifndef NDEBUG
unsigned i;
bool incomplete = false;
BITSET_FOREACH_SET(i, st.worklist, st.count) {
bi_print_instr(st.instructions[i], stderr);
incomplete = true;
}
if (incomplete)
unreachable("The above instructions failed to schedule.");
#endif
bi_free_worklist(st);
}
static bool
bi_check_fau_src(bi_instr *ins, unsigned s, uint32_t *constants, unsigned *cwords, bi_index *fau)
{
assert(s < ins->nr_srcs);
bi_index src = ins->src[s];
/* Staging registers can't have FAU accesses */
if (bi_is_staging_src(ins, s))
return (src.type != BI_INDEX_CONSTANT) && (src.type != BI_INDEX_FAU);
if (src.type == BI_INDEX_CONSTANT) {
/* Allow fast zero */
if (src.value == 0 && bi_opcode_props[ins->op].fma && bi_reads_zero(ins))
return true;
if (!bi_is_null(*fau))
return false;
/* Else, try to inline a constant */
for (unsigned i = 0; i < *cwords; ++i) {
if (src.value == constants[i])
return true;
}
if (*cwords >= 2)
return false;
constants[(*cwords)++] = src.value;
} else if (src.type == BI_INDEX_FAU) {
if (*cwords != 0)
return false;
/* Can only read from one pair of FAU words */
if (!bi_is_null(*fau) && (src.value != fau->value))
return false;
/* If there is a target, we'll need a PC-relative constant */
if (ins->branch_target)
return false;
*fau = src;
}
return true;
}
void
bi_lower_fau(bi_context *ctx)
{
bi_foreach_instr_global_safe(ctx, ins) {
bi_builder b = bi_init_builder(ctx, bi_before_instr(ins));
uint32_t constants[2];
unsigned cwords = 0;
bi_index fau = bi_null();
/* ATEST must have the ATEST datum encoded, not any other
* uniform. See to it this is the case. */
if (ins->op == BI_OPCODE_ATEST)
fau = ins->src[2];
/* Dual texturing requires the texture operation descriptor
* encoded as an immediate so we can fix up.
*/
if (ins->op == BI_OPCODE_TEXC_DUAL) {
assert(ins->src[3].type == BI_INDEX_CONSTANT);
constants[cwords++] = ins->src[3].value;
}
/* Phis get split up into moves so are unrestricted */
if (ins->op == BI_OPCODE_PHI)
continue;
bi_foreach_src(ins, s) {
if (bi_check_fau_src(ins, s, constants, &cwords, &fau)) continue;
bi_index copy = bi_mov_i32(&b, ins->src[s]);
bi_replace_src(ins, s, copy);
}
}
}
/* Only v7 allows specifying a dependency on the tilebuffer for the first
* clause of a shader. v6 requires adding a NOP clause with the depedency. */
static void
bi_add_nop_for_atest(bi_context *ctx)
{
/* Only needed on v6 */
if (ctx->arch >= 7)
return;
if (list_is_empty(&ctx->blocks))
return;
/* Fetch the first clause of the shader */
bi_block *block = list_first_entry(&ctx->blocks, bi_block, link);
bi_clause *clause = bi_next_clause(ctx, block, NULL);
if (!clause || !(clause->dependencies & ((1 << BIFROST_SLOT_ELDEST_DEPTH) |
(1 << BIFROST_SLOT_ELDEST_COLOUR))))
return;
/* Add a NOP so we can wait for the dependencies required by the first
* clause */
bi_instr *I = rzalloc(ctx, bi_instr);
I->op = BI_OPCODE_NOP;
bi_clause *new_clause = ralloc(ctx, bi_clause);
*new_clause = (bi_clause) {
.flow_control = BIFROST_FLOW_NBTB,
.next_clause_prefetch = true,
.block = clause->block,
.tuple_count = 1,
.tuples[0] = { .fma = I, },
};
list_add(&new_clause->link, &clause->block->clauses);
}
void
bi_schedule(bi_context *ctx)
{
/* Fed into both scheduling and DCE */
bi_postra_liveness(ctx);
bi_foreach_block(ctx, block) {
bi_schedule_block(ctx, block);
}
bi_opt_dce_post_ra(ctx);
bi_add_nop_for_atest(ctx);
}