| /* | |
| * Copyright © 2017 Connor Abbott | |
| * | |
| * Permission is hereby granted, free of charge, to any person obtaining a | |
| * copy of this software and associated documentation files (the "Software"), | |
| * to deal in the Software without restriction, including without limitation | |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| * and/or sell copies of the Software, and to permit persons to whom the | |
| * Software is furnished to do so, subject to the following conditions: | |
| * | |
| * The above copyright notice and this permission notice (including the next | |
| * paragraph) shall be included in all copies or substantial portions of the | |
| * Software. | |
| * | |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
| * IN THE SOFTWARE. | |
| */ | |
| #include "nir_serialize.h" | |
| #include "util/u_dynarray.h" | |
| #include "util/u_math.h" | |
| #include "util/u_printf.h" | |
| #include "nir_control_flow.h" | |
| #include "nir_xfb_info.h" | |
| #define NIR_SERIALIZE_FUNC_HAS_IMPL ((void *)(intptr_t)1) | |
| #define MAX_OBJECT_IDS (1 << 20) | |
| typedef struct { | |
| size_t blob_offset; | |
| nir_def *src; | |
| nir_block *block; | |
| } write_phi_fixup; | |
| typedef struct { | |
| const nir_shader *nir; | |
| struct blob *blob; | |
| /* maps pointer to index */ | |
| struct hash_table remap_table; | |
| /* the next index to assign to a NIR in-memory object */ | |
| uint32_t next_idx; | |
| /* Array of write_phi_fixup structs representing phi sources that need to | |
| * be resolved in the second pass. | |
| */ | |
| struct util_dynarray phi_fixups; | |
| /* The last serialized type. */ | |
| const struct glsl_type *last_type; | |
| const struct glsl_type *last_interface_type; | |
| struct nir_variable_data last_var_data; | |
| /* For skipping equal ALU headers (typical after scalarization). */ | |
| nir_instr_type last_instr_type; | |
| uintptr_t last_alu_header_offset; | |
| uint32_t last_alu_header; | |
| /* Don't write optional data such as variable names. */ | |
| bool strip; | |
| bool debug_info; | |
| } write_ctx; | |
| typedef struct { | |
| nir_shader *nir; | |
| nir_function_impl *impl; | |
| struct blob_reader *blob; | |
| /* the next index to assign to a NIR in-memory object */ | |
| uint32_t next_idx; | |
| /* The length of the index -> object table */ | |
| uint32_t idx_table_len; | |
| /* map from index to deserialized pointer */ | |
| void **idx_table; | |
| /* List of phi sources. */ | |
| struct list_head phi_srcs; | |
| /* The last deserialized type. */ | |
| const struct glsl_type *last_type; | |
| const struct glsl_type *last_interface_type; | |
| struct nir_variable_data last_var_data; | |
| struct hash_table strings; | |
| } read_ctx; | |
| static void | |
| write_add_object(write_ctx *ctx, const void *obj) | |
| { | |
| uint32_t index = ctx->next_idx++; | |
| assert(index != MAX_OBJECT_IDS); | |
| _mesa_hash_table_insert(&ctx->remap_table, obj, (void *)(uintptr_t)index); | |
| } | |
| static uint32_t | |
| write_lookup_object(write_ctx *ctx, const void *obj) | |
| { | |
| struct hash_entry *entry = _mesa_hash_table_search(&ctx->remap_table, obj); | |
| assert(entry); | |
| return (uint32_t)(uintptr_t)entry->data; | |
| } | |
| static void | |
| read_add_object(read_ctx *ctx, void *obj) | |
| { | |
| assert(ctx->next_idx < ctx->idx_table_len); | |
| ctx->idx_table[ctx->next_idx++] = obj; | |
| } | |
| static void * | |
| read_lookup_object(read_ctx *ctx, uint32_t idx) | |
| { | |
| assert(idx < ctx->idx_table_len); | |
| return ctx->idx_table[idx]; | |
| } | |
| static void * | |
| read_object(read_ctx *ctx) | |
| { | |
| return read_lookup_object(ctx, blob_read_uint32(ctx->blob)); | |
| } | |
| static uint32_t | |
| encode_bit_size_3bits(uint8_t bit_size) | |
| { | |
| /* Encode values of 0, 1, 2, 4, 8, 16, 32, 64 in 3 bits. */ | |
| assert(bit_size <= 64 && util_is_power_of_two_or_zero(bit_size)); | |
| if (bit_size) | |
| return util_logbase2(bit_size) + 1; | |
| return 0; | |
| } | |
| static uint8_t | |
| decode_bit_size_3bits(uint8_t bit_size) | |
| { | |
| if (bit_size) | |
| return 1 << (bit_size - 1); | |
| return 0; | |
| } | |
| #define NUM_COMPONENTS_IS_SEPARATE_7 7 | |
| static uint8_t | |
| encode_num_components_in_3bits(uint8_t num_components) | |
| { | |
| if (num_components <= 4) | |
| return num_components; | |
| if (num_components == 8) | |
| return 5; | |
| if (num_components == 16) | |
| return 6; | |
| /* special value indicating that num_components is in the next uint32 */ | |
| return NUM_COMPONENTS_IS_SEPARATE_7; | |
| } | |
| static uint8_t | |
| decode_num_components_in_3bits(uint8_t value) | |
| { | |
| if (value <= 4) | |
| return value; | |
| if (value == 5) | |
| return 8; | |
| if (value == 6) | |
| return 16; | |
| UNREACHABLE("invalid num_components encoding"); | |
| return 0; | |
| } | |
| static void | |
| write_constant(write_ctx *ctx, const nir_constant *c) | |
| { | |
| blob_write_bytes(ctx->blob, c->values, sizeof(c->values)); | |
| blob_write_uint32(ctx->blob, c->num_elements); | |
| for (unsigned i = 0; i < c->num_elements; i++) | |
| write_constant(ctx, c->elements[i]); | |
| } | |
| static nir_constant * | |
| read_constant(read_ctx *ctx, void *mem_ctx) | |
| { | |
| nir_constant *c = ralloc(mem_ctx, nir_constant); | |
| static const nir_const_value zero_vals[ARRAY_SIZE(c->values)] = { 0 }; | |
| blob_copy_bytes(ctx->blob, (uint8_t *)c->values, sizeof(c->values)); | |
| c->is_null_constant = memcmp(c->values, zero_vals, sizeof(c->values)) == 0; | |
| c->num_elements = blob_read_uint32(ctx->blob); | |
| if (c->num_elements) { | |
| c->elements = ralloc_array(c, nir_constant *, c->num_elements); | |
| for (unsigned i = 0; i < c->num_elements; i++) { | |
| c->elements[i] = read_constant(ctx, c); | |
| c->is_null_constant &= c->elements[i]->is_null_constant; | |
| } | |
| } else { | |
| c->elements = NULL; | |
| } | |
| return c; | |
| } | |
| enum var_data_encoding { | |
| var_encode_full, | |
| var_encode_location_diff, | |
| }; | |
| union packed_var { | |
| uint32_t u32; | |
| struct { | |
| unsigned has_name : 1; | |
| unsigned has_constant_initializer : 1; | |
| unsigned has_pointer_initializer : 1; | |
| unsigned has_interface_type : 1; | |
| unsigned num_state_slots : 7; | |
| unsigned data_encoding : 2; | |
| unsigned type_same_as_last : 1; | |
| unsigned interface_type_same_as_last : 1; | |
| unsigned ray_query : 1; | |
| unsigned num_members : 16; | |
| } u; | |
| }; | |
| union packed_var_data_diff { | |
| uint32_t u32; | |
| struct { | |
| int location : 13; | |
| int location_frac : 3; | |
| int driver_location : 16; | |
| } u; | |
| }; | |
| static void | |
| write_variable(write_ctx *ctx, const nir_variable *var) | |
| { | |
| write_add_object(ctx, var); | |
| assert(var->num_state_slots < (1 << 7)); | |
| STATIC_ASSERT(sizeof(union packed_var) == 4); | |
| union packed_var flags; | |
| flags.u32 = 0; | |
| flags.u.has_name = !ctx->strip && var->name; | |
| flags.u.has_constant_initializer = !!(var->constant_initializer); | |
| flags.u.has_pointer_initializer = !!(var->pointer_initializer); | |
| flags.u.has_interface_type = !!(var->interface_type); | |
| flags.u.type_same_as_last = var->type == ctx->last_type; | |
| flags.u.interface_type_same_as_last = | |
| var->interface_type && var->interface_type == ctx->last_interface_type; | |
| flags.u.num_state_slots = var->num_state_slots; | |
| flags.u.num_members = var->num_members; | |
| struct nir_variable_data data = var->data; | |
| /* When stripping, we expect that the location is no longer needed, | |
| * which is typically after shaders are linked. | |
| */ | |
| if (ctx->strip && | |
| data.mode != nir_var_system_value && | |
| data.mode != nir_var_shader_in && | |
| data.mode != nir_var_shader_out) | |
| data.location = 0; | |
| struct nir_variable_data tmp = data; | |
| tmp.location = ctx->last_var_data.location; | |
| tmp.location_frac = ctx->last_var_data.location_frac; | |
| tmp.driver_location = ctx->last_var_data.driver_location; | |
| /* See if we can encode only the difference in locations from the last | |
| * variable. | |
| */ | |
| if (memcmp(&ctx->last_var_data, &tmp, sizeof(tmp)) == 0 && | |
| abs((int)data.location - | |
| (int)ctx->last_var_data.location) < (1 << 12) && | |
| abs((int)data.driver_location - | |
| (int)ctx->last_var_data.driver_location) < (1 << 15)) | |
| flags.u.data_encoding = var_encode_location_diff; | |
| else | |
| flags.u.data_encoding = var_encode_full; | |
| flags.u.ray_query = var->data.ray_query; | |
| blob_write_uint32(ctx->blob, flags.u32); | |
| if (!flags.u.type_same_as_last) { | |
| encode_type_to_blob(ctx->blob, var->type); | |
| ctx->last_type = var->type; | |
| } | |
| if (var->interface_type && !flags.u.interface_type_same_as_last) { | |
| encode_type_to_blob(ctx->blob, var->interface_type); | |
| ctx->last_interface_type = var->interface_type; | |
| } | |
| if (flags.u.has_name) | |
| blob_write_string(ctx->blob, var->name); | |
| if (flags.u.data_encoding == var_encode_full) { | |
| blob_write_bytes(ctx->blob, &data, sizeof(data)); | |
| } else { | |
| /* Serialize only the difference in locations from the last variable. | |
| */ | |
| union packed_var_data_diff diff; | |
| diff.u.location = data.location - ctx->last_var_data.location; | |
| diff.u.location_frac = data.location_frac - | |
| ctx->last_var_data.location_frac; | |
| diff.u.driver_location = data.driver_location - | |
| ctx->last_var_data.driver_location; | |
| blob_write_uint32(ctx->blob, diff.u32); | |
| } | |
| ctx->last_var_data = data; | |
| for (unsigned i = 0; i < var->num_state_slots; i++) { | |
| blob_write_bytes(ctx->blob, &var->state_slots[i], | |
| sizeof(var->state_slots[i])); | |
| } | |
| if (var->constant_initializer) | |
| write_constant(ctx, var->constant_initializer); | |
| if (var->pointer_initializer) | |
| blob_write_uint32(ctx->blob, | |
| write_lookup_object(ctx, var->pointer_initializer)); | |
| if (var->num_members > 0) { | |
| blob_write_bytes(ctx->blob, (uint8_t *)var->members, | |
| var->num_members * sizeof(*var->members)); | |
| } | |
| } | |
| static nir_variable * | |
| read_variable(read_ctx *ctx) | |
| { | |
| nir_variable *var = nir_variable_create_zeroed(ctx->nir); | |
| read_add_object(ctx, var); | |
| union packed_var flags; | |
| flags.u32 = blob_read_uint32(ctx->blob); | |
| if (flags.u.type_same_as_last) { | |
| var->type = ctx->last_type; | |
| } else { | |
| var->type = decode_type_from_blob(ctx->blob); | |
| ctx->last_type = var->type; | |
| } | |
| if (flags.u.has_interface_type) { | |
| if (flags.u.interface_type_same_as_last) { | |
| var->interface_type = ctx->last_interface_type; | |
| } else { | |
| var->interface_type = decode_type_from_blob(ctx->blob); | |
| ctx->last_interface_type = var->interface_type; | |
| } | |
| } | |
| if (flags.u.has_name) { | |
| const char *name = blob_read_string(ctx->blob); | |
| nir_variable_set_name(ctx->nir, var, name); | |
| } else { | |
| var->name = NULL; | |
| } | |
| if (flags.u.data_encoding == var_encode_full) { | |
| blob_copy_bytes(ctx->blob, (uint8_t *)&var->data, sizeof(var->data)); | |
| ctx->last_var_data = var->data; | |
| } else { /* var_encode_location_diff */ | |
| union packed_var_data_diff diff; | |
| diff.u32 = blob_read_uint32(ctx->blob); | |
| var->data = ctx->last_var_data; | |
| var->data.location += diff.u.location; | |
| var->data.location_frac += diff.u.location_frac; | |
| var->data.driver_location += diff.u.driver_location; | |
| ctx->last_var_data = var->data; | |
| } | |
| var->data.ray_query = flags.u.ray_query; | |
| var->num_state_slots = flags.u.num_state_slots; | |
| if (var->num_state_slots != 0) { | |
| var->state_slots = ralloc_array(ctx->nir, nir_state_slot, | |
| var->num_state_slots); | |
| for (unsigned i = 0; i < var->num_state_slots; i++) { | |
| blob_copy_bytes(ctx->blob, &var->state_slots[i], | |
| sizeof(var->state_slots[i])); | |
| } | |
| } | |
| if (flags.u.has_constant_initializer) | |
| var->constant_initializer = read_constant(ctx, ctx->nir); | |
| else | |
| var->constant_initializer = NULL; | |
| if (flags.u.has_pointer_initializer) | |
| var->pointer_initializer = read_object(ctx); | |
| else | |
| var->pointer_initializer = NULL; | |
| var->num_members = flags.u.num_members; | |
| if (var->num_members > 0) { | |
| var->members = ralloc_array(ctx->nir, struct nir_variable_data, | |
| var->num_members); | |
| blob_copy_bytes(ctx->blob, (uint8_t *)var->members, | |
| var->num_members * sizeof(*var->members)); | |
| } | |
| return var; | |
| } | |
| static void | |
| write_var_list(write_ctx *ctx, const struct exec_list *src) | |
| { | |
| blob_write_uint32(ctx->blob, exec_list_length(src)); | |
| foreach_list_typed(nir_variable, var, node, src) { | |
| write_variable(ctx, var); | |
| } | |
| } | |
| static void | |
| read_var_list(read_ctx *ctx, struct exec_list *dst) | |
| { | |
| exec_list_make_empty(dst); | |
| unsigned num_vars = blob_read_uint32(ctx->blob); | |
| for (unsigned i = 0; i < num_vars; i++) { | |
| nir_variable *var = read_variable(ctx); | |
| exec_list_push_tail(dst, &var->node); | |
| } | |
| } | |
| union packed_src { | |
| uint32_t u32; | |
| struct { | |
| unsigned _pad : 2; /* <-- Header */ | |
| unsigned object_idx : 20; | |
| unsigned _footer : 10; /* <-- Footer */ | |
| } any; | |
| struct { | |
| unsigned _header : 22; /* <-- Header */ | |
| unsigned _pad : 2; /* <-- Footer */ | |
| unsigned swizzle_x : 2; | |
| unsigned swizzle_y : 2; | |
| unsigned swizzle_z : 2; | |
| unsigned swizzle_w : 2; | |
| } alu; | |
| struct { | |
| unsigned _header : 22; /* <-- Header */ | |
| unsigned src_type : 5; /* <-- Footer */ | |
| unsigned _pad : 5; | |
| } tex; | |
| }; | |
| static void | |
| write_src_full(write_ctx *ctx, const nir_src *src, union packed_src header) | |
| { | |
| header.any.object_idx = write_lookup_object(ctx, src->ssa); | |
| blob_write_uint32(ctx->blob, header.u32); | |
| } | |
| static void | |
| write_src(write_ctx *ctx, const nir_src *src) | |
| { | |
| union packed_src header = { 0 }; | |
| write_src_full(ctx, src, header); | |
| } | |
| static union packed_src | |
| read_src(read_ctx *ctx, nir_src *src) | |
| { | |
| STATIC_ASSERT(sizeof(union packed_src) == 4); | |
| union packed_src header; | |
| header.u32 = blob_read_uint32(ctx->blob); | |
| src->ssa = read_lookup_object(ctx, header.any.object_idx); | |
| return header; | |
| } | |
| union packed_def { | |
| uint8_t u8; | |
| struct { | |
| uint8_t _pad : 2; | |
| uint8_t num_components : 3; | |
| uint8_t bit_size : 3; | |
| }; | |
| }; | |
| enum intrinsic_const_indices_encoding { | |
| /* Use packed_const_indices to store tightly packed indices. | |
| * | |
| * The common case for load_ubo is 0, 0, 0, which is trivially represented. | |
| * The common cases for load_interpolated_input also fit here, e.g.: 7, 3 | |
| */ | |
| const_indices_all_combined, | |
| const_indices_8bit, /* 8 bits per element */ | |
| const_indices_16bit, /* 16 bits per element */ | |
| const_indices_32bit, /* 32 bits per element */ | |
| }; | |
| enum load_const_packing { | |
| /* Constants are not packed and are stored in following dwords. */ | |
| load_const_full, | |
| /* packed_value contains high 19 bits, low bits are 0, | |
| * good for floating-point decimals | |
| */ | |
| load_const_scalar_hi_19bits, | |
| /* packed_value contains low 19 bits, high bits are sign-extended */ | |
| load_const_scalar_lo_19bits_sext, | |
| }; | |
| union packed_instr { | |
| uint32_t u32; | |
| struct { | |
| unsigned instr_type : 4; /* always present */ | |
| unsigned _pad : 20; | |
| unsigned def : 8; /* always last */ | |
| } any; | |
| struct { | |
| unsigned instr_type : 4; | |
| unsigned pad : 1; | |
| unsigned no_signed_wrap : 1; | |
| unsigned no_unsigned_wrap : 1; | |
| /* Swizzles for 2 srcs */ | |
| unsigned two_swizzles : 4; | |
| unsigned op : 10; | |
| unsigned packed_src_ssa_16bit : 1; | |
| /* Scalarized ALUs always have the same header. */ | |
| unsigned num_followup_alu_sharing_header : 2; | |
| unsigned def : 8; | |
| } alu; | |
| struct { | |
| unsigned instr_type : 4; | |
| unsigned deref_type : 3; | |
| unsigned cast_type_same_as_last : 1; | |
| unsigned modes : 6; /* See (de|en)code_deref_modes() */ | |
| unsigned _pad : 8; | |
| unsigned in_bounds : 1; | |
| unsigned packed_src_ssa_16bit : 1; /* deref_var redefines this */ | |
| unsigned def : 8; | |
| } deref; | |
| struct { | |
| unsigned instr_type : 4; | |
| unsigned deref_type : 3; | |
| unsigned _pad : 1; | |
| unsigned object_idx : 16; /* if 0, the object ID is a separate uint32 */ | |
| unsigned def : 8; | |
| } deref_var; | |
| struct { | |
| unsigned instr_type : 4; | |
| unsigned intrinsic : 10; | |
| unsigned const_indices_encoding : 2; | |
| unsigned packed_const_indices : 8; | |
| unsigned def : 8; | |
| } intrinsic; | |
| struct { | |
| unsigned instr_type : 4; | |
| unsigned last_component : 4; | |
| unsigned bit_size : 3; | |
| unsigned packing : 2; /* enum load_const_packing */ | |
| unsigned packed_value : 19; /* meaning determined by packing */ | |
| } load_const; | |
| struct { | |
| unsigned instr_type : 4; | |
| unsigned last_component : 4; | |
| unsigned bit_size : 3; | |
| unsigned _pad : 21; | |
| } undef; | |
| struct { | |
| unsigned instr_type : 4; | |
| unsigned num_srcs : 4; | |
| unsigned op : 6; | |
| unsigned _pad : 10; | |
| unsigned def : 8; | |
| } tex; | |
| struct { | |
| unsigned instr_type : 4; | |
| unsigned num_srcs : 20; | |
| unsigned def : 8; | |
| } phi; | |
| struct { | |
| unsigned instr_type : 4; | |
| unsigned type : 2; | |
| unsigned _pad : 26; | |
| } jump; | |
| struct { | |
| unsigned instr_type : 4; | |
| unsigned type : 4; | |
| unsigned string_length : 16; | |
| unsigned def : 8; | |
| } debug_info; | |
| }; | |
| /* Write "lo24" as low 24 bits in the first uint32. */ | |
| static void | |
| write_def(write_ctx *ctx, const nir_def *def, union packed_instr header, | |
| nir_instr_type instr_type) | |
| { | |
| STATIC_ASSERT(sizeof(union packed_def) == 1); | |
| union packed_def pdef; | |
| pdef.u8 = 0; | |
| pdef.num_components = | |
| encode_num_components_in_3bits(def->num_components); | |
| pdef.bit_size = encode_bit_size_3bits(def->bit_size); | |
| header.any.def = pdef.u8; | |
| /* Check if the current ALU instruction has the same header as the previous | |
| * instruction that is also ALU. If it is, we don't have to write | |
| * the current header. This is a typical occurence after scalarization. | |
| */ | |
| if (instr_type == nir_instr_type_alu && likely(!ctx->debug_info)) { | |
| bool equal_header = false; | |
| if (ctx->last_instr_type == nir_instr_type_alu) { | |
| assert(ctx->last_alu_header_offset); | |
| union packed_instr last_header; | |
| last_header.u32 = ctx->last_alu_header; | |
| /* Clear the field that counts ALUs with equal headers. */ | |
| union packed_instr clean_header; | |
| clean_header.u32 = last_header.u32; | |
| clean_header.alu.num_followup_alu_sharing_header = 0; | |
| /* There can be at most 4 consecutive ALU instructions | |
| * sharing the same header. | |
| */ | |
| if (last_header.alu.num_followup_alu_sharing_header < 3 && | |
| header.u32 == clean_header.u32) { | |
| last_header.alu.num_followup_alu_sharing_header++; | |
| blob_overwrite_uint32(ctx->blob, ctx->last_alu_header_offset, | |
| last_header.u32); | |
| ctx->last_alu_header = last_header.u32; | |
| equal_header = true; | |
| } | |
| } | |
| if (!equal_header) { | |
| ctx->last_alu_header_offset = blob_reserve_uint32(ctx->blob); | |
| blob_overwrite_uint32(ctx->blob, ctx->last_alu_header_offset, header.u32); | |
| ctx->last_alu_header = header.u32; | |
| } | |
| } else { | |
| blob_write_uint32(ctx->blob, header.u32); | |
| } | |
| if (pdef.num_components == NUM_COMPONENTS_IS_SEPARATE_7) | |
| blob_write_uint32(ctx->blob, def->num_components); | |
| write_add_object(ctx, def); | |
| } | |
| static void | |
| read_def(read_ctx *ctx, nir_def *def, nir_instr *instr, | |
| union packed_instr header) | |
| { | |
| union packed_def pdef; | |
| pdef.u8 = header.any.def; | |
| unsigned bit_size = decode_bit_size_3bits(pdef.bit_size); | |
| unsigned num_components; | |
| if (pdef.num_components == NUM_COMPONENTS_IS_SEPARATE_7) | |
| num_components = blob_read_uint32(ctx->blob); | |
| else | |
| num_components = decode_num_components_in_3bits(pdef.num_components); | |
| nir_def_init(instr, def, num_components, bit_size); | |
| read_add_object(ctx, def); | |
| } | |
| static bool | |
| are_object_ids_16bit(write_ctx *ctx) | |
| { | |
| /* Check the highest object ID, because they are monotonic. */ | |
| return ctx->next_idx < (1 << 16); | |
| } | |
| static bool | |
| is_alu_src_ssa_16bit(write_ctx *ctx, const nir_alu_instr *alu) | |
| { | |
| unsigned num_srcs = nir_op_infos[alu->op].num_inputs; | |
| for (unsigned i = 0; i < num_srcs; i++) { | |
| unsigned src_components = nir_ssa_alu_instr_src_components(alu, i); | |
| for (unsigned chan = 0; chan < src_components; chan++) { | |
| /* The swizzles for src0.x and src1.x are stored | |
| * in two_swizzles for SSA ALUs. | |
| */ | |
| if (i < 2 && chan == 0 && alu->src[i].swizzle[chan] < 4) | |
| continue; | |
| if (alu->src[i].swizzle[chan] != chan) | |
| return false; | |
| } | |
| } | |
| return are_object_ids_16bit(ctx); | |
| } | |
| static void | |
| write_alu(write_ctx *ctx, const nir_alu_instr *alu) | |
| { | |
| unsigned num_srcs = nir_op_infos[alu->op].num_inputs; | |
| /* 10 bits for nir_op */ | |
| STATIC_ASSERT(nir_num_opcodes <= 1024); | |
| union packed_instr header; | |
| header.u32 = 0; | |
| header.alu.instr_type = alu->instr.type; | |
| header.alu.no_signed_wrap = alu->no_signed_wrap; | |
| header.alu.no_unsigned_wrap = alu->no_unsigned_wrap; | |
| header.alu.op = alu->op; | |
| header.alu.packed_src_ssa_16bit = is_alu_src_ssa_16bit(ctx, alu); | |
| if (header.alu.packed_src_ssa_16bit) { | |
| /* For packed srcs of SSA ALUs, this field stores the swizzles. */ | |
| header.alu.two_swizzles = alu->src[0].swizzle[0]; | |
| if (num_srcs > 1) | |
| header.alu.two_swizzles |= alu->src[1].swizzle[0] << 2; | |
| } | |
| write_def(ctx, &alu->def, header, alu->instr.type); | |
| if (nir_op_infos[alu->op].valid_fp_math_ctrl) | |
| blob_write_uint32(ctx->blob, alu->fp_math_ctrl); | |
| if (header.alu.packed_src_ssa_16bit) { | |
| for (unsigned i = 0; i < num_srcs; i++) { | |
| unsigned idx = write_lookup_object(ctx, alu->src[i].src.ssa); | |
| assert(idx < (1 << 16)); | |
| blob_write_uint16(ctx->blob, idx); | |
| } | |
| } else { | |
| for (unsigned i = 0; i < num_srcs; i++) { | |
| unsigned src_channels = nir_ssa_alu_instr_src_components(alu, i); | |
| unsigned src_components = nir_src_num_components(alu->src[i].src); | |
| union packed_src src; | |
| bool packed = src_components <= 4 && src_channels <= 4; | |
| src.u32 = 0; | |
| if (packed) { | |
| src.alu.swizzle_x = alu->src[i].swizzle[0]; | |
| if (src_channels >= 2) | |
| src.alu.swizzle_y = alu->src[i].swizzle[1]; | |
| if (src_channels >= 3) | |
| src.alu.swizzle_z = alu->src[i].swizzle[2]; | |
| if (src_channels >= 4) | |
| src.alu.swizzle_w = alu->src[i].swizzle[3]; | |
| } | |
| write_src_full(ctx, &alu->src[i].src, src); | |
| /* Store swizzles for vec8 and vec16. */ | |
| if (!packed) { | |
| for (unsigned o = 0; o < src_channels; o += 8) { | |
| unsigned value = 0; | |
| for (unsigned j = 0; j < 8 && o + j < src_channels; j++) { | |
| value |= (uint32_t)alu->src[i].swizzle[o + j] << (4 * j); /* 4 bits per swizzle */ | |
| } | |
| blob_write_uint32(ctx->blob, value); | |
| } | |
| } | |
| } | |
| } | |
| } | |
| static nir_alu_instr * | |
| read_alu(read_ctx *ctx, union packed_instr header) | |
| { | |
| unsigned num_srcs = nir_op_infos[header.alu.op].num_inputs; | |
| nir_alu_instr *alu = nir_alu_instr_create(ctx->nir, header.alu.op); | |
| alu->no_signed_wrap = header.alu.no_signed_wrap; | |
| alu->no_unsigned_wrap = header.alu.no_unsigned_wrap; | |
| read_def(ctx, &alu->def, &alu->instr, header); | |
| if (nir_op_infos[alu->op].valid_fp_math_ctrl) | |
| alu->fp_math_ctrl = blob_read_uint32(ctx->blob); | |
| if (header.alu.packed_src_ssa_16bit) { | |
| for (unsigned i = 0; i < num_srcs; i++) { | |
| nir_alu_src *src = &alu->src[i]; | |
| src->src.ssa = read_lookup_object(ctx, blob_read_uint16(ctx->blob)); | |
| memset(&src->swizzle, 0, sizeof(src->swizzle)); | |
| unsigned src_components = nir_ssa_alu_instr_src_components(alu, i); | |
| for (unsigned chan = 0; chan < src_components; chan++) | |
| src->swizzle[chan] = chan; | |
| } | |
| } else { | |
| for (unsigned i = 0; i < num_srcs; i++) { | |
| union packed_src src = read_src(ctx, &alu->src[i].src); | |
| unsigned src_channels = nir_ssa_alu_instr_src_components(alu, i); | |
| unsigned src_components = nir_src_num_components(alu->src[i].src); | |
| bool packed = src_components <= 4 && src_channels <= 4; | |
| memset(&alu->src[i].swizzle, 0, sizeof(alu->src[i].swizzle)); | |
| if (packed) { | |
| alu->src[i].swizzle[0] = src.alu.swizzle_x; | |
| if (src_channels >= 2) | |
| alu->src[i].swizzle[1] = src.alu.swizzle_y; | |
| if (src_channels >= 3) | |
| alu->src[i].swizzle[2] = src.alu.swizzle_z; | |
| if (src_channels >= 4) | |
| alu->src[i].swizzle[3] = src.alu.swizzle_w; | |
| } else { | |
| /* Load swizzles for vec8 and vec16. */ | |
| for (unsigned o = 0; o < src_channels; o += 8) { | |
| unsigned value = blob_read_uint32(ctx->blob); | |
| for (unsigned j = 0; j < 8 && o + j < src_channels; j++) { | |
| alu->src[i].swizzle[o + j] = | |
| (value >> (4 * j)) & 0xf; /* 4 bits per swizzle */ | |
| } | |
| } | |
| } | |
| } | |
| } | |
| if (header.alu.packed_src_ssa_16bit) { | |
| alu->src[0].swizzle[0] = header.alu.two_swizzles & 0x3; | |
| if (num_srcs > 1) | |
| alu->src[1].swizzle[0] = header.alu.two_swizzles >> 2; | |
| } | |
| return alu; | |
| } | |
| #define NUM_GENERIC_MODES 4 | |
| #define MODE_ENC_GENERIC_BIT (1 << 5) | |
| static nir_variable_mode | |
| decode_deref_modes(unsigned modes) | |
| { | |
| if (modes & MODE_ENC_GENERIC_BIT) { | |
| modes &= ~MODE_ENC_GENERIC_BIT; | |
| return modes << (ffs(nir_var_mem_generic) - 1); | |
| } else { | |
| return 1 << modes; | |
| } | |
| } | |
| static unsigned | |
| encode_deref_modes(nir_variable_mode modes) | |
| { | |
| /* Mode sets on derefs generally come in two forms. For certain OpenCL | |
| * cases, we can have more than one of the generic modes set. In this | |
| * case, we need the full bitfield. Fortunately, there are only 4 of | |
| * these. For all other modes, we can only have one mode at a time so we | |
| * can compress them by only storing the bit position. This, plus one bit | |
| * to select encoding, lets us pack the entire bitfield in 6 bits. | |
| */ | |
| /* Assert that the modes we are compressing fit along with the generic bit | |
| */ | |
| STATIC_ASSERT((nir_num_variable_modes - NUM_GENERIC_MODES) < | |
| MODE_ENC_GENERIC_BIT); | |
| /* Assert that the generic modes are defined at the end of the modes enum | |
| */ | |
| STATIC_ASSERT((nir_var_all & ~nir_var_mem_generic) < | |
| (1 << (nir_num_variable_modes - NUM_GENERIC_MODES))); | |
| unsigned enc; | |
| if (modes == 0 || (modes & nir_var_mem_generic)) { | |
| assert(!(modes & ~nir_var_mem_generic)); | |
| enc = modes >> (ffs(nir_var_mem_generic) - 1); | |
| assert(enc < MODE_ENC_GENERIC_BIT); | |
| enc |= MODE_ENC_GENERIC_BIT; | |
| } else { | |
| assert(util_is_power_of_two_nonzero(modes)); | |
| enc = ffs(modes) - 1; | |
| assert(enc < MODE_ENC_GENERIC_BIT); | |
| } | |
| assert(modes == decode_deref_modes(enc)); | |
| return enc; | |
| } | |
| static void | |
| write_deref(write_ctx *ctx, const nir_deref_instr *deref) | |
| { | |
| assert(deref->deref_type < 8); | |
| union packed_instr header; | |
| header.u32 = 0; | |
| header.deref.instr_type = deref->instr.type; | |
| header.deref.deref_type = deref->deref_type; | |
| if (deref->deref_type == nir_deref_type_cast) { | |
| header.deref.modes = encode_deref_modes(deref->modes); | |
| header.deref.cast_type_same_as_last = deref->type == ctx->last_type; | |
| } | |
| unsigned var_idx = 0; | |
| if (deref->deref_type == nir_deref_type_var) { | |
| var_idx = write_lookup_object(ctx, deref->var); | |
| if (var_idx && var_idx < (1 << 16)) | |
| header.deref_var.object_idx = var_idx; | |
| } | |
| if (nir_deref_instr_is_arr(deref)) { | |
| header.deref.packed_src_ssa_16bit = are_object_ids_16bit(ctx); | |
| header.deref.in_bounds = deref->arr.in_bounds; | |
| } | |
| write_def(ctx, &deref->def, header, deref->instr.type); | |
| switch (deref->deref_type) { | |
| case nir_deref_type_var: | |
| if (!header.deref_var.object_idx) | |
| blob_write_uint32(ctx->blob, var_idx); | |
| break; | |
| case nir_deref_type_struct: | |
| write_src(ctx, &deref->parent); | |
| blob_write_uint32(ctx->blob, deref->strct.index); | |
| break; | |
| case nir_deref_type_array: | |
| case nir_deref_type_ptr_as_array: | |
| if (header.deref.packed_src_ssa_16bit) { | |
| blob_write_uint16(ctx->blob, | |
| write_lookup_object(ctx, deref->parent.ssa)); | |
| blob_write_uint16(ctx->blob, | |
| write_lookup_object(ctx, deref->arr.index.ssa)); | |
| } else { | |
| write_src(ctx, &deref->parent); | |
| write_src(ctx, &deref->arr.index); | |
| } | |
| break; | |
| case nir_deref_type_cast: | |
| write_src(ctx, &deref->parent); | |
| blob_write_uint32(ctx->blob, deref->cast.ptr_stride); | |
| blob_write_uint32(ctx->blob, deref->cast.align_mul); | |
| blob_write_uint32(ctx->blob, deref->cast.align_offset); | |
| if (!header.deref.cast_type_same_as_last) { | |
| encode_type_to_blob(ctx->blob, deref->type); | |
| ctx->last_type = deref->type; | |
| } | |
| break; | |
| case nir_deref_type_array_wildcard: | |
| write_src(ctx, &deref->parent); | |
| break; | |
| default: | |
| UNREACHABLE("Invalid deref type"); | |
| } | |
| } | |
| static nir_deref_instr * | |
| read_deref(read_ctx *ctx, union packed_instr header) | |
| { | |
| nir_deref_type deref_type = header.deref.deref_type; | |
| nir_deref_instr *deref = nir_deref_instr_create(ctx->nir, deref_type); | |
| read_def(ctx, &deref->def, &deref->instr, header); | |
| nir_deref_instr *parent; | |
| switch (deref->deref_type) { | |
| case nir_deref_type_var: | |
| if (header.deref_var.object_idx) | |
| deref->var = read_lookup_object(ctx, header.deref_var.object_idx); | |
| else | |
| deref->var = read_object(ctx); | |
| deref->type = deref->var->type; | |
| break; | |
| case nir_deref_type_struct: | |
| read_src(ctx, &deref->parent); | |
| parent = nir_src_as_deref(deref->parent); | |
| deref->strct.index = blob_read_uint32(ctx->blob); | |
| deref->type = glsl_get_struct_field(parent->type, deref->strct.index); | |
| break; | |
| case nir_deref_type_array: | |
| case nir_deref_type_ptr_as_array: | |
| if (header.deref.packed_src_ssa_16bit) { | |
| deref->parent.ssa = read_lookup_object(ctx, blob_read_uint16(ctx->blob)); | |
| deref->arr.index.ssa = read_lookup_object(ctx, blob_read_uint16(ctx->blob)); | |
| } else { | |
| read_src(ctx, &deref->parent); | |
| read_src(ctx, &deref->arr.index); | |
| } | |
| deref->arr.in_bounds = header.deref.in_bounds; | |
| parent = nir_src_as_deref(deref->parent); | |
| if (deref->deref_type == nir_deref_type_array) | |
| deref->type = glsl_get_array_element(parent->type); | |
| else | |
| deref->type = parent->type; | |
| break; | |
| case nir_deref_type_cast: | |
| read_src(ctx, &deref->parent); | |
| deref->cast.ptr_stride = blob_read_uint32(ctx->blob); | |
| deref->cast.align_mul = blob_read_uint32(ctx->blob); | |
| deref->cast.align_offset = blob_read_uint32(ctx->blob); | |
| if (header.deref.cast_type_same_as_last) { | |
| deref->type = ctx->last_type; | |
| } else { | |
| deref->type = decode_type_from_blob(ctx->blob); | |
| ctx->last_type = deref->type; | |
| } | |
| break; | |
| case nir_deref_type_array_wildcard: | |
| read_src(ctx, &deref->parent); | |
| parent = nir_src_as_deref(deref->parent); | |
| deref->type = glsl_get_array_element(parent->type); | |
| break; | |
| default: | |
| UNREACHABLE("Invalid deref type"); | |
| } | |
| if (deref_type == nir_deref_type_var) { | |
| deref->modes = deref->var->data.mode; | |
| } else if (deref->deref_type == nir_deref_type_cast) { | |
| deref->modes = decode_deref_modes(header.deref.modes); | |
| } else { | |
| deref->modes = nir_def_as_deref(deref->parent.ssa)->modes; | |
| } | |
| return deref; | |
| } | |
| static void | |
| write_intrinsic(write_ctx *ctx, const nir_intrinsic_instr *intrin) | |
| { | |
| /* 10 bits for nir_intrinsic_op */ | |
| STATIC_ASSERT(nir_num_intrinsics <= 1024); | |
| unsigned num_srcs = nir_intrinsic_infos[intrin->intrinsic].num_srcs; | |
| unsigned num_index_slots = nir_intrinsic_infos[intrin->intrinsic].num_index_slots; | |
| assert(intrin->intrinsic < 1024); | |
| union packed_instr header; | |
| header.u32 = 0; | |
| header.intrinsic.instr_type = intrin->instr.type; | |
| header.intrinsic.intrinsic = intrin->intrinsic; | |
| /* Analyze constant indices to decide how to encode them. */ | |
| if (num_index_slots) { | |
| unsigned max_bits = 0; | |
| for (unsigned i = 0; i < num_index_slots; i++) { | |
| unsigned max = util_last_bit(intrin->const_index[i]); | |
| max_bits = MAX2(max_bits, max); | |
| } | |
| if (max_bits * num_index_slots <= 8) { | |
| header.intrinsic.const_indices_encoding = const_indices_all_combined; | |
| /* Pack all const indices into 8 bits. */ | |
| unsigned bit_size = 8 / num_index_slots; | |
| for (unsigned i = 0; i < num_index_slots; i++) { | |
| header.intrinsic.packed_const_indices |= | |
| intrin->const_index[i] << (i * bit_size); | |
| } | |
| } else if (max_bits <= 8) | |
| header.intrinsic.const_indices_encoding = const_indices_8bit; | |
| else if (max_bits <= 16) | |
| header.intrinsic.const_indices_encoding = const_indices_16bit; | |
| else | |
| header.intrinsic.const_indices_encoding = const_indices_32bit; | |
| } | |
| if (nir_intrinsic_infos[intrin->intrinsic].has_dest) | |
| write_def(ctx, &intrin->def, header, intrin->instr.type); | |
| else | |
| blob_write_uint32(ctx->blob, header.u32); | |
| for (unsigned i = 0; i < num_srcs; i++) | |
| write_src(ctx, &intrin->src[i]); | |
| if (num_index_slots) { | |
| switch (header.intrinsic.const_indices_encoding) { | |
| case const_indices_8bit: | |
| for (unsigned i = 0; i < num_index_slots; i++) | |
| blob_write_uint8(ctx->blob, intrin->const_index[i]); | |
| break; | |
| case const_indices_16bit: | |
| for (unsigned i = 0; i < num_index_slots; i++) | |
| blob_write_uint16(ctx->blob, intrin->const_index[i]); | |
| break; | |
| case const_indices_32bit: | |
| for (unsigned i = 0; i < num_index_slots; i++) | |
| blob_write_uint32(ctx->blob, intrin->const_index[i]); | |
| break; | |
| } | |
| } | |
| } | |
| static nir_intrinsic_instr * | |
| read_intrinsic(read_ctx *ctx, union packed_instr header) | |
| { | |
| nir_intrinsic_op op = header.intrinsic.intrinsic; | |
| nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(ctx->nir, op); | |
| unsigned num_srcs = nir_intrinsic_infos[op].num_srcs; | |
| unsigned num_index_slots = nir_intrinsic_infos[op].num_index_slots; | |
| if (nir_intrinsic_infos[op].has_dest) | |
| read_def(ctx, &intrin->def, &intrin->instr, header); | |
| for (unsigned i = 0; i < num_srcs; i++) | |
| read_src(ctx, &intrin->src[i]); | |
| /* Vectorized instrinsics have num_components same as dst or src that has | |
| * 0 components in the info. Find it. | |
| */ | |
| if (nir_intrinsic_infos[op].has_dest && | |
| nir_intrinsic_infos[op].dest_components == 0) { | |
| intrin->num_components = intrin->def.num_components; | |
| } else { | |
| for (unsigned i = 0; i < num_srcs; i++) { | |
| if (nir_intrinsic_infos[op].src_components[i] == 0) { | |
| intrin->num_components = nir_src_num_components(intrin->src[i]); | |
| break; | |
| } | |
| } | |
| } | |
| if (num_index_slots) { | |
| switch (header.intrinsic.const_indices_encoding) { | |
| case const_indices_all_combined: { | |
| unsigned bit_size = 8 / num_index_slots; | |
| unsigned bit_mask = u_bit_consecutive(0, bit_size); | |
| for (unsigned i = 0; i < num_index_slots; i++) { | |
| intrin->const_index[i] = | |
| (header.intrinsic.packed_const_indices >> (i * bit_size)) & | |
| bit_mask; | |
| } | |
| break; | |
| } | |
| case const_indices_8bit: | |
| for (unsigned i = 0; i < num_index_slots; i++) | |
| intrin->const_index[i] = blob_read_uint8(ctx->blob); | |
| break; | |
| case const_indices_16bit: | |
| for (unsigned i = 0; i < num_index_slots; i++) | |
| intrin->const_index[i] = blob_read_uint16(ctx->blob); | |
| break; | |
| case const_indices_32bit: | |
| for (unsigned i = 0; i < num_index_slots; i++) | |
| intrin->const_index[i] = blob_read_uint32(ctx->blob); | |
| break; | |
| } | |
| } | |
| return intrin; | |
| } | |
| static void | |
| write_load_const(write_ctx *ctx, const nir_load_const_instr *lc) | |
| { | |
| assert(lc->def.num_components >= 1 && lc->def.num_components <= 16); | |
| union packed_instr header; | |
| header.u32 = 0; | |
| header.load_const.instr_type = lc->instr.type; | |
| header.load_const.last_component = lc->def.num_components - 1; | |
| header.load_const.bit_size = encode_bit_size_3bits(lc->def.bit_size); | |
| header.load_const.packing = load_const_full; | |
| /* Try to pack 1-component constants into the 19 free bits in the header. */ | |
| if (lc->def.num_components == 1) { | |
| switch (lc->def.bit_size) { | |
| case 64: | |
| if ((lc->value[0].u64 & 0x1fffffffffffull) == 0) { | |
| /* packed_value contains high 19 bits, low bits are 0 */ | |
| header.load_const.packing = load_const_scalar_hi_19bits; | |
| header.load_const.packed_value = lc->value[0].u64 >> 45; | |
| } else if (util_mask_sign_extend(lc->value[0].i64, 19) == lc->value[0].i64) { | |
| /* packed_value contains low 19 bits, high bits are sign-extended */ | |
| header.load_const.packing = load_const_scalar_lo_19bits_sext; | |
| header.load_const.packed_value = lc->value[0].u64; | |
| } | |
| break; | |
| case 32: | |
| if ((lc->value[0].u32 & 0x1fff) == 0) { | |
| header.load_const.packing = load_const_scalar_hi_19bits; | |
| header.load_const.packed_value = lc->value[0].u32 >> 13; | |
| } else if (util_mask_sign_extend(lc->value[0].i32, 19) == lc->value[0].i32) { | |
| header.load_const.packing = load_const_scalar_lo_19bits_sext; | |
| header.load_const.packed_value = lc->value[0].u32; | |
| } | |
| break; | |
| case 16: | |
| header.load_const.packing = load_const_scalar_lo_19bits_sext; | |
| header.load_const.packed_value = lc->value[0].u16; | |
| break; | |
| case 8: | |
| header.load_const.packing = load_const_scalar_lo_19bits_sext; | |
| header.load_const.packed_value = lc->value[0].u8; | |
| break; | |
| case 1: | |
| header.load_const.packing = load_const_scalar_lo_19bits_sext; | |
| header.load_const.packed_value = lc->value[0].b; | |
| break; | |
| default: | |
| UNREACHABLE("invalid bit_size"); | |
| } | |
| } | |
| blob_write_uint32(ctx->blob, header.u32); | |
| if (header.load_const.packing == load_const_full) { | |
| switch (lc->def.bit_size) { | |
| case 64: | |
| blob_write_bytes(ctx->blob, lc->value, | |
| sizeof(*lc->value) * lc->def.num_components); | |
| break; | |
| case 32: | |
| for (unsigned i = 0; i < lc->def.num_components; i++) | |
| blob_write_uint32(ctx->blob, lc->value[i].u32); | |
| break; | |
| case 16: | |
| for (unsigned i = 0; i < lc->def.num_components; i++) | |
| blob_write_uint16(ctx->blob, lc->value[i].u16); | |
| break; | |
| default: | |
| assert(lc->def.bit_size <= 8); | |
| for (unsigned i = 0; i < lc->def.num_components; i++) | |
| blob_write_uint8(ctx->blob, lc->value[i].u8); | |
| break; | |
| } | |
| } | |
| write_add_object(ctx, &lc->def); | |
| } | |
| static nir_load_const_instr * | |
| read_load_const(read_ctx *ctx, union packed_instr header) | |
| { | |
| nir_load_const_instr *lc = | |
| nir_load_const_instr_create(ctx->nir, header.load_const.last_component + 1, | |
| decode_bit_size_3bits(header.load_const.bit_size)); | |
| switch (header.load_const.packing) { | |
| case load_const_scalar_hi_19bits: | |
| switch (lc->def.bit_size) { | |
| case 64: | |
| lc->value[0].u64 = (uint64_t)header.load_const.packed_value << 45; | |
| break; | |
| case 32: | |
| lc->value[0].u32 = (uint64_t)header.load_const.packed_value << 13; | |
| break; | |
| default: | |
| UNREACHABLE("invalid bit_size"); | |
| } | |
| break; | |
| case load_const_scalar_lo_19bits_sext: | |
| switch (lc->def.bit_size) { | |
| case 64: | |
| lc->value[0].u64 = header.load_const.packed_value; | |
| if (lc->value[0].u64 >> 18) | |
| lc->value[0].u64 |= UINT64_C(0xfffffffffff80000); | |
| break; | |
| case 32: | |
| lc->value[0].u32 = header.load_const.packed_value; | |
| if (lc->value[0].u32 >> 18) | |
| lc->value[0].u32 |= 0xfff80000; | |
| break; | |
| case 16: | |
| lc->value[0].u16 = header.load_const.packed_value; | |
| break; | |
| case 8: | |
| lc->value[0].u8 = header.load_const.packed_value; | |
| break; | |
| case 1: | |
| lc->value[0].b = header.load_const.packed_value; | |
| break; | |
| default: | |
| UNREACHABLE("invalid bit_size"); | |
| } | |
| break; | |
| case load_const_full: | |
| switch (lc->def.bit_size) { | |
| case 64: | |
| blob_copy_bytes(ctx->blob, lc->value, sizeof(*lc->value) * lc->def.num_components); | |
| break; | |
| case 32: | |
| for (unsigned i = 0; i < lc->def.num_components; i++) | |
| lc->value[i].u32 = blob_read_uint32(ctx->blob); | |
| break; | |
| case 16: | |
| for (unsigned i = 0; i < lc->def.num_components; i++) | |
| lc->value[i].u16 = blob_read_uint16(ctx->blob); | |
| break; | |
| default: | |
| assert(lc->def.bit_size <= 8); | |
| for (unsigned i = 0; i < lc->def.num_components; i++) | |
| lc->value[i].u8 = blob_read_uint8(ctx->blob); | |
| break; | |
| } | |
| break; | |
| } | |
| read_add_object(ctx, &lc->def); | |
| return lc; | |
| } | |
| static void | |
| write_ssa_undef(write_ctx *ctx, const nir_undef_instr *undef) | |
| { | |
| assert(undef->def.num_components >= 1 && undef->def.num_components <= 16); | |
| union packed_instr header; | |
| header.u32 = 0; | |
| header.undef.instr_type = undef->instr.type; | |
| header.undef.last_component = undef->def.num_components - 1; | |
| header.undef.bit_size = encode_bit_size_3bits(undef->def.bit_size); | |
| blob_write_uint32(ctx->blob, header.u32); | |
| write_add_object(ctx, &undef->def); | |
| } | |
| static nir_undef_instr * | |
| read_ssa_undef(read_ctx *ctx, union packed_instr header) | |
| { | |
| nir_undef_instr *undef = | |
| nir_undef_instr_create(ctx->nir, header.undef.last_component + 1, | |
| decode_bit_size_3bits(header.undef.bit_size)); | |
| read_add_object(ctx, &undef->def); | |
| return undef; | |
| } | |
| union packed_tex_data { | |
| uint32_t u32; | |
| struct { | |
| unsigned sampler_dim : 4; | |
| unsigned dest_type : 8; | |
| unsigned coord_components : 3; | |
| unsigned is_array : 1; | |
| unsigned is_shadow : 1; | |
| unsigned is_new_style_shadow : 1; | |
| unsigned is_sparse : 1; | |
| unsigned component : 2; | |
| unsigned texture_non_uniform : 1; | |
| unsigned sampler_non_uniform : 1; | |
| unsigned embedded_sampler : 1; | |
| unsigned array_is_lowered_cube : 1; | |
| unsigned is_gather_implicit_lod : 1; | |
| unsigned can_speculate : 1; | |
| unsigned input_attachment_index : 1; | |
| unsigned input_attachment_depth : 1; | |
| unsigned input_attachment_stencil : 1; | |
| unsigned unused : 2; /* Mark unused for valgrind. */ | |
| } u; | |
| }; | |
| union packed_tex2_data { | |
| uint32_t u32; | |
| struct { | |
| unsigned texture_2_non_uniform : 1; | |
| unsigned sampler_2_non_uniform : 1; | |
| unsigned embedded_sampler_2 : 1; | |
| unsigned unused : 29; /* Mark unused for valgrind. */ | |
| } u; | |
| }; | |
| static void | |
| write_tex(write_ctx *ctx, const nir_tex_instr *tex) | |
| { | |
| assert(tex->num_srcs < 16); | |
| assert(tex->op < 64); | |
| union packed_instr header; | |
| header.u32 = 0; | |
| header.tex.instr_type = tex->instr.type; | |
| header.tex.num_srcs = tex->num_srcs; | |
| header.tex.op = tex->op; | |
| write_def(ctx, &tex->def, header, tex->instr.type); | |
| blob_write_uint32(ctx->blob, tex->texture_index); | |
| blob_write_uint32(ctx->blob, tex->texture_array_size); | |
| blob_write_uint32(ctx->blob, tex->sampler_index); | |
| if (nir_tex_instr_need_texture_2(tex)) | |
| blob_write_uint32(ctx->blob, tex->sampler_2_index); | |
| blob_write_uint32(ctx->blob, tex->backend_flags); | |
| if (tex->op == nir_texop_tg4) | |
| blob_write_bytes(ctx->blob, tex->tg4_offsets, sizeof(tex->tg4_offsets)); | |
| STATIC_ASSERT(sizeof(union packed_tex_data) == sizeof(uint32_t)); | |
| union packed_tex_data packed = { | |
| .u.sampler_dim = tex->sampler_dim, | |
| .u.dest_type = tex->dest_type, | |
| .u.coord_components = tex->coord_components, | |
| .u.is_array = tex->is_array, | |
| .u.is_shadow = tex->is_shadow, | |
| .u.is_new_style_shadow = tex->is_new_style_shadow, | |
| .u.is_sparse = tex->is_sparse, | |
| .u.component = tex->component, | |
| .u.texture_non_uniform = tex->texture_non_uniform, | |
| .u.sampler_non_uniform = tex->sampler_non_uniform, | |
| .u.embedded_sampler = tex->embedded_sampler, | |
| .u.array_is_lowered_cube = tex->array_is_lowered_cube, | |
| .u.is_gather_implicit_lod = tex->is_gather_implicit_lod, | |
| .u.can_speculate = tex->can_speculate, | |
| .u.input_attachment_index = tex->input_attachment_index, | |
| .u.input_attachment_depth = tex->input_attachment_depth, | |
| .u.input_attachment_stencil = tex->input_attachment_stencil, | |
| }; | |
| blob_write_uint32(ctx->blob, packed.u32); | |
| if (nir_tex_instr_need_texture_2(tex)) { | |
| STATIC_ASSERT(sizeof(union packed_tex2_data) == sizeof(uint32_t)); | |
| union packed_tex2_data packed = { | |
| .u.embedded_sampler_2 = tex->embedded_sampler_2, | |
| .u.texture_2_non_uniform = tex->texture_2_non_uniform, | |
| .u.sampler_2_non_uniform = tex->sampler_2_non_uniform, | |
| }; | |
| blob_write_uint32(ctx->blob, packed.u32); | |
| } | |
| for (unsigned i = 0; i < tex->num_srcs; i++) { | |
| union packed_src src; | |
| src.u32 = 0; | |
| src.tex.src_type = tex->src[i].src_type; | |
| write_src_full(ctx, &tex->src[i].src, src); | |
| } | |
| } | |
| static nir_tex_instr * | |
| read_tex(read_ctx *ctx, union packed_instr header) | |
| { | |
| nir_tex_instr *tex = nir_tex_instr_create(ctx->nir, header.tex.num_srcs); | |
| read_def(ctx, &tex->def, &tex->instr, header); | |
| tex->op = header.tex.op; | |
| tex->texture_index = blob_read_uint32(ctx->blob); | |
| tex->texture_array_size = blob_read_uint32(ctx->blob); | |
| tex->sampler_index = blob_read_uint32(ctx->blob); | |
| if (nir_tex_instr_need_texture_2(tex)) | |
| tex->sampler_2_index = blob_read_uint32(ctx->blob); | |
| tex->backend_flags = blob_read_uint32(ctx->blob); | |
| if (tex->op == nir_texop_tg4) | |
| blob_copy_bytes(ctx->blob, tex->tg4_offsets, sizeof(tex->tg4_offsets)); | |
| union packed_tex_data packed; | |
| packed.u32 = blob_read_uint32(ctx->blob); | |
| tex->sampler_dim = packed.u.sampler_dim; | |
| tex->dest_type = packed.u.dest_type; | |
| tex->coord_components = packed.u.coord_components; | |
| tex->is_array = packed.u.is_array; | |
| tex->is_shadow = packed.u.is_shadow; | |
| tex->is_new_style_shadow = packed.u.is_new_style_shadow; | |
| tex->is_sparse = packed.u.is_sparse; | |
| tex->component = packed.u.component; | |
| tex->texture_non_uniform = packed.u.texture_non_uniform; | |
| tex->sampler_non_uniform = packed.u.sampler_non_uniform; | |
| tex->embedded_sampler = packed.u.embedded_sampler; | |
| tex->array_is_lowered_cube = packed.u.array_is_lowered_cube; | |
| tex->is_gather_implicit_lod = packed.u.is_gather_implicit_lod; | |
| tex->can_speculate = packed.u.can_speculate; | |
| tex->input_attachment_index = packed.u.input_attachment_index; | |
| tex->input_attachment_depth = packed.u.input_attachment_depth; | |
| tex->input_attachment_stencil = packed.u.input_attachment_stencil; | |
| if (nir_tex_instr_need_texture_2(tex)) { | |
| union packed_tex2_data packed; | |
| packed.u32 = blob_read_uint32(ctx->blob); | |
| tex->embedded_sampler_2 = packed.u.embedded_sampler_2; | |
| tex->texture_2_non_uniform = packed.u.texture_2_non_uniform; | |
| tex->sampler_2_non_uniform = packed.u.sampler_2_non_uniform; | |
| } | |
| for (unsigned i = 0; i < tex->num_srcs; i++) { | |
| union packed_src src = read_src(ctx, &tex->src[i].src); | |
| tex->src[i].src_type = src.tex.src_type; | |
| } | |
| return tex; | |
| } | |
| static void | |
| write_phi(write_ctx *ctx, const nir_phi_instr *phi) | |
| { | |
| union packed_instr header; | |
| header.u32 = 0; | |
| header.phi.instr_type = phi->instr.type; | |
| header.phi.num_srcs = exec_list_length(&phi->srcs); | |
| /* Phi nodes are special, since they may reference SSA definitions and | |
| * basic blocks that don't exist yet. We leave two empty uint32_t's here, | |
| * and then store enough information so that a later fixup pass can fill | |
| * them in correctly. | |
| */ | |
| write_def(ctx, &phi->def, header, phi->instr.type); | |
| nir_foreach_phi_src(src, phi) { | |
| size_t blob_offset = blob_reserve_uint32(ctx->blob); | |
| ASSERTED size_t blob_offset2 = blob_reserve_uint32(ctx->blob); | |
| assert(blob_offset + sizeof(uint32_t) == blob_offset2); | |
| write_phi_fixup fixup = { | |
| .blob_offset = blob_offset, | |
| .src = src->src.ssa, | |
| .block = src->pred, | |
| }; | |
| util_dynarray_append(&ctx->phi_fixups, fixup); | |
| } | |
| } | |
| static void | |
| write_fixup_phis(write_ctx *ctx) | |
| { | |
| util_dynarray_foreach(&ctx->phi_fixups, write_phi_fixup, fixup) { | |
| blob_overwrite_uint32(ctx->blob, fixup->blob_offset, | |
| write_lookup_object(ctx, fixup->src)); | |
| blob_overwrite_uint32(ctx->blob, fixup->blob_offset + sizeof(uint32_t), | |
| write_lookup_object(ctx, fixup->block)); | |
| } | |
| util_dynarray_clear(&ctx->phi_fixups); | |
| } | |
| static nir_phi_instr * | |
| read_phi(read_ctx *ctx, nir_block *blk, union packed_instr header) | |
| { | |
| nir_phi_instr *phi = nir_phi_instr_create(ctx->nir); | |
| read_def(ctx, &phi->def, &phi->instr, header); | |
| /* For similar reasons as before, we just store the index directly into the | |
| * pointer, and let a later pass resolve the phi sources. | |
| * | |
| * In order to ensure that the copied sources (which are just the indices | |
| * from the blob for now) don't get inserted into the old shader's use-def | |
| * lists, we have to add the phi instruction *before* we set up its | |
| * sources. | |
| */ | |
| nir_instr_insert_after_block(blk, &phi->instr); | |
| for (unsigned i = 0; i < header.phi.num_srcs; i++) { | |
| nir_def *def = (nir_def *)(uintptr_t)blob_read_uint32(ctx->blob); | |
| nir_block *pred = (nir_block *)(uintptr_t)blob_read_uint32(ctx->blob); | |
| nir_phi_src *src = nir_phi_instr_add_src(phi, pred, def); | |
| /* Since we're not letting nir_insert_instr handle use/def stuff for us, | |
| * we have to set the parent_instr manually. It doesn't really matter | |
| * when we do it, so we might as well do it here. | |
| */ | |
| nir_src_set_use_instr(&src->src, &phi->instr); | |
| /* Stash it in the list of phi sources. We'll walk this list and fix up | |
| * sources at the very end of read_function_impl. | |
| */ | |
| list_add(&src->src.use_link, &ctx->phi_srcs); | |
| } | |
| return phi; | |
| } | |
| static void | |
| read_fixup_phis(read_ctx *ctx) | |
| { | |
| list_for_each_entry_safe(nir_phi_src, src, &ctx->phi_srcs, src.use_link) { | |
| src->pred = read_lookup_object(ctx, (uintptr_t)src->pred); | |
| src->src.ssa = read_lookup_object(ctx, (uintptr_t)src->src.ssa); | |
| /* Remove from this list */ | |
| list_del(&src->src.use_link); | |
| list_addtail(&src->src.use_link, &src->src.ssa->uses); | |
| } | |
| assert(list_is_empty(&ctx->phi_srcs)); | |
| } | |
| static void | |
| write_jump(write_ctx *ctx, const nir_jump_instr *jmp) | |
| { | |
| /* These aren't handled because they require special block linking */ | |
| assert(jmp->type != nir_jump_goto && jmp->type != nir_jump_goto_if); | |
| assert(jmp->type < 4); | |
| union packed_instr header; | |
| header.u32 = 0; | |
| header.jump.instr_type = jmp->instr.type; | |
| header.jump.type = jmp->type; | |
| blob_write_uint32(ctx->blob, header.u32); | |
| } | |
| static nir_jump_instr * | |
| read_jump(read_ctx *ctx, union packed_instr header) | |
| { | |
| /* These aren't handled because they require special block linking */ | |
| assert(header.jump.type != nir_jump_goto && | |
| header.jump.type != nir_jump_goto_if); | |
| nir_jump_instr *jmp = nir_jump_instr_create(ctx->nir, header.jump.type); | |
| return jmp; | |
| } | |
| static void | |
| write_call(write_ctx *ctx, const nir_call_instr *call) | |
| { | |
| blob_write_uint32(ctx->blob, write_lookup_object(ctx, call->callee)); | |
| for (unsigned i = 0; i < call->num_params; i++) | |
| write_src(ctx, &call->params[i]); | |
| } | |
| static nir_call_instr * | |
| read_call(read_ctx *ctx) | |
| { | |
| nir_function *callee = read_object(ctx); | |
| nir_call_instr *call = nir_call_instr_create(ctx->nir, callee); | |
| for (unsigned i = 0; i < call->num_params; i++) | |
| read_src(ctx, &call->params[i]); | |
| return call; | |
| } | |
| static void | |
| write_cmat_call(write_ctx *ctx, const nir_cmat_call_instr *call) | |
| { | |
| if (call->callee) | |
| blob_write_uint32(ctx->blob, write_lookup_object(ctx, call->callee)); | |
| else | |
| blob_write_uint32(ctx->blob, 0); | |
| blob_write_uint32(ctx->blob, call->op); | |
| for (unsigned i = 0; i < call->num_params; i++) | |
| write_src(ctx, &call->params[i]); | |
| for (unsigned i = 0; i < NIR_CMAT_CALL_MAX_CONST_INDEX; i++) | |
| blob_write_uint32(ctx->blob, call->const_index[i]); | |
| } | |
| static nir_cmat_call_instr * | |
| read_cmat_call(read_ctx *ctx) | |
| { | |
| uint32_t callee_int = blob_read_uint32(ctx->blob); | |
| nir_function *callee = NULL; | |
| if (callee_int) | |
| callee = read_lookup_object(ctx, callee_int); | |
| nir_cmat_call_op op = blob_read_uint32(ctx->blob); | |
| nir_cmat_call_instr *call = nir_cmat_call_instr_create(ctx->nir, op, callee); | |
| for (unsigned i = 0; i < call->num_params; i++) | |
| read_src(ctx, &call->params[i]); | |
| for (unsigned i = 0; i < NIR_CMAT_CALL_MAX_CONST_INDEX; i++) | |
| call->const_index[i] = blob_read_uint32(ctx->blob); | |
| return call; | |
| } | |
| enum nir_serialize_debug_info_flags { | |
| NIR_SERIALIZE_FILENAME = 1 << 0, | |
| NIR_SERIALIZE_VARIABLE_NAME = 1 << 1, | |
| }; | |
| static void | |
| write_debug_info(write_ctx *ctx, const nir_instr *instr) | |
| { | |
| nir_instr_debug_info *debug_info = nir_instr_get_debug_info((void *)instr); | |
| blob_write_uint32(ctx->blob, debug_info->line); | |
| blob_write_uint32(ctx->blob, debug_info->column); | |
| blob_write_uint32(ctx->blob, debug_info->spirv_offset); | |
| blob_write_uint32(ctx->blob, debug_info->nir_line); | |
| enum nir_serialize_debug_info_flags flags = 0; | |
| if (debug_info->filename) | |
| flags |= NIR_SERIALIZE_FILENAME; | |
| if (debug_info->variable_name) | |
| flags |= NIR_SERIALIZE_VARIABLE_NAME; | |
| blob_write_uint8(ctx->blob, flags); | |
| if (debug_info->filename) | |
| blob_write_string(ctx->blob, debug_info->filename); | |
| if (debug_info->variable_name) | |
| blob_write_string(ctx->blob, debug_info->variable_name); | |
| } | |
| static void | |
| read_debug_info(read_ctx *ctx, nir_instr_debug_info *debug_info) | |
| { | |
| memset(debug_info, 0, sizeof(*debug_info)); | |
| debug_info->line = blob_read_uint32(ctx->blob); | |
| debug_info->column = blob_read_uint32(ctx->blob); | |
| debug_info->spirv_offset = blob_read_uint32(ctx->blob); | |
| debug_info->nir_line = blob_read_uint32(ctx->blob); | |
| enum nir_serialize_debug_info_flags flags = blob_read_uint8(ctx->blob); | |
| if (flags & NIR_SERIALIZE_FILENAME) { | |
| const char *filename = blob_read_string(ctx->blob); | |
| struct hash_entry *entry = _mesa_hash_table_search(&ctx->strings, filename); | |
| if (entry) { | |
| debug_info->filename = entry->data; | |
| } else { | |
| debug_info->filename = ralloc_strdup(ctx->nir, filename); | |
| _mesa_hash_table_insert(&ctx->strings, filename, debug_info->filename); | |
| } | |
| } | |
| if (flags & NIR_SERIALIZE_VARIABLE_NAME) { | |
| const char *variable_name = blob_read_string(ctx->blob); | |
| struct hash_entry *entry = _mesa_hash_table_search(&ctx->strings, variable_name); | |
| if (entry) { | |
| debug_info->variable_name = entry->data; | |
| } else { | |
| debug_info->variable_name = ralloc_strdup(ctx->nir, variable_name); | |
| _mesa_hash_table_insert(&ctx->strings, variable_name, debug_info->variable_name); | |
| } | |
| } | |
| } | |
| static void | |
| write_instr(write_ctx *ctx, const nir_instr *instr) | |
| { | |
| /* We have only 4 bits for the instruction type. */ | |
| assert(instr->type < 16); | |
| if (unlikely(ctx->debug_info)) | |
| write_debug_info(ctx, instr); | |
| switch (instr->type) { | |
| case nir_instr_type_alu: | |
| write_alu(ctx, nir_instr_as_alu(instr)); | |
| break; | |
| case nir_instr_type_deref: | |
| write_deref(ctx, nir_instr_as_deref(instr)); | |
| break; | |
| case nir_instr_type_intrinsic: | |
| write_intrinsic(ctx, nir_instr_as_intrinsic(instr)); | |
| break; | |
| case nir_instr_type_load_const: | |
| write_load_const(ctx, nir_instr_as_load_const(instr)); | |
| break; | |
| case nir_instr_type_undef: | |
| write_ssa_undef(ctx, nir_instr_as_undef(instr)); | |
| break; | |
| case nir_instr_type_tex: | |
| write_tex(ctx, nir_instr_as_tex(instr)); | |
| break; | |
| case nir_instr_type_phi: | |
| write_phi(ctx, nir_instr_as_phi(instr)); | |
| break; | |
| case nir_instr_type_jump: | |
| write_jump(ctx, nir_instr_as_jump(instr)); | |
| break; | |
| case nir_instr_type_call: | |
| blob_write_uint32(ctx->blob, instr->type); | |
| write_call(ctx, nir_instr_as_call(instr)); | |
| break; | |
| case nir_instr_type_cmat_call: | |
| blob_write_uint32(ctx->blob, instr->type); | |
| write_cmat_call(ctx, nir_instr_as_cmat_call(instr)); | |
| break; | |
| default: | |
| UNREACHABLE("bad instr type"); | |
| } | |
| } | |
| /* Return the number of instructions read. */ | |
| static unsigned | |
| read_instr(read_ctx *ctx, nir_block *block) | |
| { | |
| nir_instr_debug_info debug_info; | |
| if (unlikely(ctx->nir->has_debug_info)) | |
| read_debug_info(ctx, &debug_info); | |
| STATIC_ASSERT(sizeof(union packed_instr) == 4); | |
| union packed_instr header; | |
| header.u32 = blob_read_uint32(ctx->blob); | |
| nir_instr *instr; | |
| switch (header.any.instr_type) { | |
| case nir_instr_type_alu: | |
| for (unsigned i = 0; i <= header.alu.num_followup_alu_sharing_header; i++) | |
| nir_instr_insert_after_block(block, &read_alu(ctx, header)->instr); | |
| return header.alu.num_followup_alu_sharing_header + 1; | |
| case nir_instr_type_deref: | |
| instr = &read_deref(ctx, header)->instr; | |
| break; | |
| case nir_instr_type_intrinsic: | |
| instr = &read_intrinsic(ctx, header)->instr; | |
| break; | |
| case nir_instr_type_load_const: | |
| instr = &read_load_const(ctx, header)->instr; | |
| break; | |
| case nir_instr_type_undef: | |
| instr = &read_ssa_undef(ctx, header)->instr; | |
| break; | |
| case nir_instr_type_tex: | |
| instr = &read_tex(ctx, header)->instr; | |
| break; | |
| case nir_instr_type_phi: | |
| /* Phi instructions are a bit of a special case when reading because we | |
| * don't want inserting the instruction to automatically handle use/defs | |
| * for us. Instead, we need to wait until all the blocks/instructions | |
| * are read so that we can set their sources up. | |
| */ | |
| read_phi(ctx, block, header); | |
| return 1; | |
| case nir_instr_type_jump: | |
| instr = &read_jump(ctx, header)->instr; | |
| break; | |
| case nir_instr_type_call: | |
| instr = &read_call(ctx)->instr; | |
| break; | |
| case nir_instr_type_cmat_call: | |
| instr = &read_cmat_call(ctx)->instr; | |
| break; | |
| default: | |
| UNREACHABLE("bad instr type"); | |
| } | |
| if (unlikely(ctx->nir->has_debug_info)) { | |
| nir_instr_debug_info *dst = nir_instr_get_debug_info(instr); | |
| memcpy(dst, &debug_info, offsetof(nir_instr_debug_info, instr)); | |
| } | |
| nir_instr_insert_after_block(block, instr); | |
| return 1; | |
| } | |
| static void | |
| write_block(write_ctx *ctx, const nir_block *block) | |
| { | |
| write_add_object(ctx, block); | |
| blob_write_uint32(ctx->blob, exec_list_length(&block->instr_list)); | |
| ctx->last_instr_type = ~0; | |
| ctx->last_alu_header_offset = 0; | |
| nir_foreach_instr(instr, block) { | |
| write_instr(ctx, instr); | |
| ctx->last_instr_type = instr->type; | |
| } | |
| } | |
| static void | |
| read_block(read_ctx *ctx, struct exec_list *cf_list) | |
| { | |
| /* Don't actually create a new block. Just use the one from the tail of | |
| * the list. NIR guarantees that the tail of the list is a block and that | |
| * no two blocks are side-by-side in the IR; It should be empty. | |
| */ | |
| nir_block *block = | |
| exec_node_data(nir_block, exec_list_get_tail(cf_list), cf_node.node); | |
| read_add_object(ctx, block); | |
| unsigned num_instrs = blob_read_uint32(ctx->blob); | |
| for (unsigned i = 0; i < num_instrs;) { | |
| i += read_instr(ctx, block); | |
| } | |
| } | |
| static void | |
| write_cf_list(write_ctx *ctx, const struct exec_list *cf_list); | |
| static void | |
| read_cf_list(read_ctx *ctx, struct exec_list *cf_list); | |
| static void | |
| write_if(write_ctx *ctx, nir_if *nif) | |
| { | |
| write_src(ctx, &nif->condition); | |
| blob_write_uint8(ctx->blob, nif->control); | |
| write_cf_list(ctx, &nif->then_list); | |
| write_cf_list(ctx, &nif->else_list); | |
| } | |
| static void | |
| read_if(read_ctx *ctx, struct exec_list *cf_list) | |
| { | |
| nir_if *nif = nir_if_create(ctx->impl); | |
| read_src(ctx, &nif->condition); | |
| nif->control = blob_read_uint8(ctx->blob); | |
| nir_cf_node_insert_end(cf_list, &nif->cf_node); | |
| read_cf_list(ctx, &nif->then_list); | |
| read_cf_list(ctx, &nif->else_list); | |
| } | |
| static void | |
| write_loop(write_ctx *ctx, nir_loop *loop) | |
| { | |
| blob_write_uint8(ctx->blob, loop->control); | |
| bool has_continue_construct = nir_loop_has_continue_construct(loop); | |
| blob_write_uint8(ctx->blob, has_continue_construct); | |
| blob_write_uint8(ctx->blob, loop->do_while); | |
| write_cf_list(ctx, &loop->body); | |
| if (has_continue_construct) { | |
| write_cf_list(ctx, &loop->continue_list); | |
| } | |
| } | |
| static void | |
| read_loop(read_ctx *ctx, struct exec_list *cf_list) | |
| { | |
| nir_loop *loop = nir_loop_create(ctx->impl); | |
| nir_cf_node_insert_end(cf_list, &loop->cf_node); | |
| loop->control = blob_read_uint8(ctx->blob); | |
| bool has_continue_construct = blob_read_uint8(ctx->blob); | |
| loop->do_while = blob_read_uint8(ctx->blob); | |
| read_cf_list(ctx, &loop->body); | |
| if (has_continue_construct) { | |
| nir_loop_add_continue_construct(loop); | |
| read_cf_list(ctx, &loop->continue_list); | |
| } | |
| } | |
| static void | |
| write_cf_node(write_ctx *ctx, nir_cf_node *cf) | |
| { | |
| blob_write_uint32(ctx->blob, cf->type); | |
| switch (cf->type) { | |
| case nir_cf_node_block: | |
| write_block(ctx, nir_cf_node_as_block(cf)); | |
| break; | |
| case nir_cf_node_if: | |
| write_if(ctx, nir_cf_node_as_if(cf)); | |
| break; | |
| case nir_cf_node_loop: | |
| write_loop(ctx, nir_cf_node_as_loop(cf)); | |
| break; | |
| default: | |
| UNREACHABLE("bad cf type"); | |
| } | |
| } | |
| static void | |
| read_cf_node(read_ctx *ctx, struct exec_list *list) | |
| { | |
| nir_cf_node_type type = blob_read_uint32(ctx->blob); | |
| switch (type) { | |
| case nir_cf_node_block: | |
| read_block(ctx, list); | |
| break; | |
| case nir_cf_node_if: | |
| read_if(ctx, list); | |
| break; | |
| case nir_cf_node_loop: | |
| read_loop(ctx, list); | |
| break; | |
| default: | |
| UNREACHABLE("bad cf type"); | |
| } | |
| } | |
| static void | |
| write_cf_list(write_ctx *ctx, const struct exec_list *cf_list) | |
| { | |
| blob_write_uint32(ctx->blob, exec_list_length(cf_list)); | |
| foreach_list_typed(nir_cf_node, cf, node, cf_list) { | |
| write_cf_node(ctx, cf); | |
| } | |
| } | |
| static void | |
| read_cf_list(read_ctx *ctx, struct exec_list *cf_list) | |
| { | |
| uint32_t num_cf_nodes = blob_read_uint32(ctx->blob); | |
| for (unsigned i = 0; i < num_cf_nodes; i++) | |
| read_cf_node(ctx, cf_list); | |
| } | |
| static void | |
| write_function_impl(write_ctx *ctx, const nir_function_impl *fi) | |
| { | |
| blob_write_uint8(ctx->blob, fi->structured); | |
| blob_write_uint8(ctx->blob, !!fi->preamble); | |
| if (fi->preamble) | |
| blob_write_uint32(ctx->blob, write_lookup_object(ctx, fi->preamble)); | |
| write_var_list(ctx, &fi->locals); | |
| write_cf_list(ctx, &fi->body); | |
| write_fixup_phis(ctx); | |
| } | |
| static nir_function_impl * | |
| read_function_impl(read_ctx *ctx) | |
| { | |
| nir_function_impl *fi = nir_function_impl_create_bare(ctx->nir); | |
| fi->structured = blob_read_uint8(ctx->blob); | |
| bool preamble = blob_read_uint8(ctx->blob); | |
| if (preamble) | |
| fi->preamble = read_object(ctx); | |
| read_var_list(ctx, &fi->locals); | |
| ctx->impl = fi; | |
| read_cf_list(ctx, &fi->body); | |
| read_fixup_phis(ctx); | |
| fi->valid_metadata = 0; | |
| return fi; | |
| } | |
| static void | |
| write_function(write_ctx *ctx, const nir_function *fxn) | |
| { | |
| uint32_t flags = 0; | |
| if (fxn->is_entrypoint) | |
| flags |= 0x1; | |
| if (fxn->is_preamble) | |
| flags |= 0x2; | |
| if (fxn->name && !ctx->strip) | |
| flags |= 0x4; | |
| if (fxn->impl) | |
| flags |= 0x8; | |
| if (fxn->should_inline) | |
| flags |= 0x10; | |
| if (fxn->dont_inline) | |
| flags |= 0x20; | |
| if (fxn->is_subroutine) | |
| flags |= 0x40; | |
| if (fxn->is_tmp_globals_wrapper) | |
| flags |= 0x80; | |
| if (fxn->workgroup_size[0] || fxn->workgroup_size[1] || fxn->workgroup_size[2]) | |
| flags |= 0x100; | |
| if (fxn->cmat_call) | |
| flags |= 0x200; | |
| blob_write_uint32(ctx->blob, flags); | |
| if (fxn->name && !ctx->strip) | |
| blob_write_string(ctx->blob, fxn->name); | |
| if (flags & 0x100) { | |
| blob_write_uint32(ctx->blob, fxn->workgroup_size[0]); | |
| blob_write_uint32(ctx->blob, fxn->workgroup_size[1]); | |
| blob_write_uint32(ctx->blob, fxn->workgroup_size[2]); | |
| } | |
| blob_write_uint32(ctx->blob, fxn->driver_attributes); | |
| blob_write_uint32(ctx->blob, fxn->subroutine_index); | |
| blob_write_uint32(ctx->blob, fxn->num_subroutine_types); | |
| for (unsigned i = 0; i < fxn->num_subroutine_types; i++) { | |
| encode_type_to_blob(ctx->blob, fxn->subroutine_types[i]); | |
| } | |
| write_add_object(ctx, fxn); | |
| blob_write_uint32(ctx->blob, fxn->num_params); | |
| for (unsigned i = 0; i < fxn->num_params; i++) { | |
| uint32_t val = | |
| ((uint32_t)fxn->params[i].num_components) | | |
| ((uint32_t)fxn->params[i].bit_size) << 8; | |
| bool has_name = fxn->params[i].name && !ctx->strip; | |
| if (has_name) | |
| val |= 0x10000; | |
| if (fxn->params[i].is_return) | |
| val |= (1u << 17); | |
| if (fxn->params[i].is_uniform) | |
| val |= (1u << 18); | |
| blob_write_uint32(ctx->blob, val); | |
| if (has_name) | |
| blob_write_string(ctx->blob, fxn->params[i].name); | |
| encode_type_to_blob(ctx->blob, fxn->params[i].type); | |
| blob_write_uint32(ctx->blob, encode_deref_modes(fxn->params[i].mode)); | |
| blob_write_uint32(ctx->blob, fxn->params[i].driver_attributes); | |
| } | |
| /* At first glance, it looks like we should write the function_impl here. | |
| * However, call instructions need to be able to reference at least the | |
| * function and those will get processed as we write the function_impls. | |
| * We stop here and write function_impls as a second pass. | |
| */ | |
| } | |
| static nir_function * | |
| read_function(read_ctx *ctx) | |
| { | |
| uint32_t flags = blob_read_uint32(ctx->blob); | |
| bool has_name = flags & 0x4; | |
| char *name = has_name ? blob_read_string(ctx->blob) : NULL; | |
| nir_function *fxn = nir_function_create(ctx->nir, name); | |
| if (flags & 0x100) { | |
| fxn->workgroup_size[0] = blob_read_uint32(ctx->blob); | |
| fxn->workgroup_size[1] = blob_read_uint32(ctx->blob); | |
| fxn->workgroup_size[2] = blob_read_uint32(ctx->blob); | |
| } | |
| fxn->driver_attributes = blob_read_uint32(ctx->blob); | |
| fxn->subroutine_index = blob_read_uint32(ctx->blob); | |
| fxn->num_subroutine_types = blob_read_uint32(ctx->blob); | |
| for (unsigned i = 0; i < fxn->num_subroutine_types; i++) { | |
| fxn->subroutine_types[i] = decode_type_from_blob(ctx->blob); | |
| } | |
| read_add_object(ctx, fxn); | |
| fxn->num_params = blob_read_uint32(ctx->blob); | |
| fxn->params = rzalloc_array(fxn, nir_parameter, fxn->num_params); | |
| for (unsigned i = 0; i < fxn->num_params; i++) { | |
| uint32_t val = blob_read_uint32(ctx->blob); | |
| bool has_name = (val & 0x10000); | |
| if (has_name) { | |
| char *name = blob_read_string(ctx->blob); | |
| fxn->params[i].name = ralloc_strdup(ctx->nir, name); | |
| } | |
| fxn->params[i].num_components = val & 0xff; | |
| fxn->params[i].bit_size = (val >> 8) & 0xff; | |
| fxn->params[i].is_return = val & (1u << 17); | |
| fxn->params[i].is_uniform = val & (1u << 18); | |
| fxn->params[i].type = decode_type_from_blob(ctx->blob); | |
| fxn->params[i].mode = decode_deref_modes(blob_read_uint32(ctx->blob)); | |
| fxn->params[i].driver_attributes = blob_read_uint32(ctx->blob); | |
| } | |
| fxn->is_entrypoint = flags & 0x1; | |
| fxn->is_preamble = flags & 0x2; | |
| if (flags & 0x8) | |
| fxn->impl = NIR_SERIALIZE_FUNC_HAS_IMPL; | |
| fxn->should_inline = flags & 0x10; | |
| fxn->dont_inline = flags & 0x20; | |
| fxn->is_subroutine = flags & 0x40; | |
| fxn->is_tmp_globals_wrapper = flags & 0x80; | |
| fxn->cmat_call = flags & 0x200; | |
| return fxn; | |
| } | |
| static void | |
| write_xfb_info(write_ctx *ctx, const nir_xfb_info *xfb) | |
| { | |
| if (xfb == NULL) { | |
| blob_write_uint32(ctx->blob, 0); | |
| } else { | |
| size_t size = nir_xfb_info_size(xfb->output_count); | |
| assert(size <= UINT32_MAX); | |
| blob_write_uint32(ctx->blob, size); | |
| blob_write_bytes(ctx->blob, xfb, size); | |
| } | |
| } | |
| static nir_xfb_info * | |
| read_xfb_info(read_ctx *ctx) | |
| { | |
| uint32_t size = blob_read_uint32(ctx->blob); | |
| if (size == 0) | |
| return NULL; | |
| struct nir_xfb_info *xfb = ralloc_size(ctx->nir, size); | |
| blob_copy_bytes(ctx->blob, (void *)xfb, size); | |
| return xfb; | |
| } | |
| enum nir_serialize_shader_flags { | |
| NIR_SERIALIZE_SHADER_NAME = 1 << 0, | |
| NIR_SERIALIZE_SHADER_LABEL = 1 << 1, | |
| NIR_SERIALIZE_DEBUG_INFO = 1 << 2, | |
| NIR_SERIALIZE_SHADER_SPEC = 1 << 3, | |
| }; | |
| void | |
| nir_serialize_function(struct blob *blob, const nir_function *fxn) | |
| { | |
| write_ctx ctx = { 0 }; | |
| _mesa_pointer_hash_table_init(&ctx.remap_table, NULL); | |
| ctx.blob = blob; | |
| ctx.nir = fxn->shader; | |
| ctx.strip = true; | |
| ctx.phi_fixups = UTIL_DYNARRAY_INIT; | |
| size_t idx_size_offset = blob_reserve_uint32(blob); | |
| write_function(&ctx, fxn); | |
| write_function_impl(&ctx, fxn->impl); | |
| blob_overwrite_uint32(blob, idx_size_offset, ctx.next_idx); | |
| _mesa_hash_table_fini(&ctx.remap_table, NULL); | |
| util_dynarray_fini(&ctx.phi_fixups); | |
| } | |
| static void | |
| serialize_internal(struct blob *blob, const nir_shader *nir, bool strip, bool serialize_info) | |
| { | |
| write_ctx ctx = { 0 }; | |
| _mesa_pointer_hash_table_init(&ctx.remap_table, NULL); | |
| ctx.blob = blob; | |
| ctx.nir = nir; | |
| ctx.strip = strip; | |
| ctx.debug_info = nir->has_debug_info && !strip; | |
| ctx.phi_fixups = UTIL_DYNARRAY_INIT; | |
| size_t idx_size_offset = blob_reserve_uint32(blob); | |
| struct shader_info info = nir->info; | |
| if (!serialize_info) | |
| memset(&info, 0, sizeof(info)); | |
| enum nir_serialize_shader_flags flags = 0; | |
| if (!strip && info.name) | |
| flags |= NIR_SERIALIZE_SHADER_NAME; | |
| if (!strip && info.label) | |
| flags |= NIR_SERIALIZE_SHADER_LABEL; | |
| if (!strip && info.spec) | |
| flags |= NIR_SERIALIZE_SHADER_SPEC; | |
| if (ctx.debug_info) | |
| flags |= NIR_SERIALIZE_DEBUG_INFO; | |
| blob_write_uint32(blob, flags); | |
| if (!strip && info.name) | |
| blob_write_string(blob, info.name); | |
| if (!strip && info.label) | |
| blob_write_string(blob, info.label); | |
| if (!strip && info.spec) | |
| blob_write_string(blob, info.spec); | |
| info.name = info.label = info.spec = NULL; | |
| blob_write_bytes(blob, (uint8_t *)&info, sizeof(info)); | |
| write_var_list(&ctx, &nir->variables); | |
| blob_write_uint32(blob, nir->num_inputs); | |
| blob_write_uint32(blob, nir->num_uniforms); | |
| blob_write_uint32(blob, nir->num_outputs); | |
| blob_write_uint32(blob, nir->scratch_size); | |
| blob_write_uint32(blob, exec_list_length(&nir->functions)); | |
| nir_foreach_function(fxn, nir) { | |
| write_function(&ctx, fxn); | |
| } | |
| nir_foreach_function_impl(impl, nir) { | |
| write_function_impl(&ctx, impl); | |
| } | |
| blob_write_uint32(blob, nir->constant_data_size); | |
| if (nir->constant_data_size > 0) | |
| blob_write_bytes(blob, nir->constant_data, nir->constant_data_size); | |
| write_xfb_info(&ctx, nir->xfb_info); | |
| if (nir->info.uses_printf) | |
| u_printf_serialize_info(blob, nir->printf_info, nir->printf_info_count); | |
| blob_overwrite_uint32(blob, idx_size_offset, ctx.next_idx); | |
| _mesa_hash_table_fini(&ctx.remap_table, NULL); | |
| util_dynarray_fini(&ctx.phi_fixups); | |
| } | |
| /** | |
| * Serialize NIR into a binary blob. | |
| * | |
| * \param strip Don't serialize information only useful for debugging, | |
| * such as variable names, making cache hits from similar | |
| * shaders more likely. | |
| */ | |
| void | |
| nir_serialize(struct blob *blob, const nir_shader *nir, bool strip) | |
| { | |
| serialize_internal(blob, nir, strip, true); | |
| } | |
| nir_shader * | |
| nir_deserialize(void *mem_ctx, | |
| const struct nir_shader_compiler_options *options, | |
| struct blob_reader *blob) | |
| { | |
| read_ctx ctx = { 0 }; | |
| ctx.blob = blob; | |
| list_inithead(&ctx.phi_srcs); | |
| ctx.idx_table_len = blob_read_uint32(blob); | |
| ctx.idx_table = calloc(ctx.idx_table_len, sizeof(uintptr_t)); | |
| enum nir_serialize_shader_flags flags = blob_read_uint32(blob); | |
| char *name = (flags & NIR_SERIALIZE_SHADER_NAME) ? blob_read_string(blob) : NULL; | |
| char *label = (flags & NIR_SERIALIZE_SHADER_LABEL) ? blob_read_string(blob) : NULL; | |
| char *spec = (flags & NIR_SERIALIZE_SHADER_SPEC) ? blob_read_string(blob) : NULL; | |
| struct shader_info info; | |
| blob_copy_bytes(blob, (uint8_t *)&info, sizeof(info)); | |
| ctx.nir = nir_shader_create(mem_ctx, info.stage, options); | |
| ctx.nir->has_debug_info = !!(flags & NIR_SERIALIZE_DEBUG_INFO); | |
| if (ctx.nir->has_debug_info) | |
| _mesa_hash_table_init(&ctx.strings, NULL, _mesa_hash_string, _mesa_key_string_equal); | |
| info.name = name ? ralloc_strdup(ctx.nir, name) : NULL; | |
| info.label = label ? ralloc_strdup(ctx.nir, label) : NULL; | |
| info.spec = spec ? ralloc_strdup(ctx.nir, spec) : NULL; | |
| ctx.nir->info = info; | |
| read_var_list(&ctx, &ctx.nir->variables); | |
| ctx.nir->num_inputs = blob_read_uint32(blob); | |
| ctx.nir->num_uniforms = blob_read_uint32(blob); | |
| ctx.nir->num_outputs = blob_read_uint32(blob); | |
| ctx.nir->scratch_size = blob_read_uint32(blob); | |
| unsigned num_functions = blob_read_uint32(blob); | |
| for (unsigned i = 0; i < num_functions; i++) | |
| read_function(&ctx); | |
| nir_foreach_function(fxn, ctx.nir) { | |
| if (fxn->impl == NIR_SERIALIZE_FUNC_HAS_IMPL) | |
| nir_function_set_impl(fxn, read_function_impl(&ctx)); | |
| } | |
| ctx.nir->constant_data_size = blob_read_uint32(blob); | |
| if (ctx.nir->constant_data_size > 0) { | |
| ctx.nir->constant_data = | |
| ralloc_size(ctx.nir, ctx.nir->constant_data_size); | |
| blob_copy_bytes(blob, ctx.nir->constant_data, | |
| ctx.nir->constant_data_size); | |
| } | |
| ctx.nir->xfb_info = read_xfb_info(&ctx); | |
| if (ctx.nir->info.uses_printf) { | |
| ctx.nir->printf_info = | |
| u_printf_deserialize_info(ctx.nir, blob, | |
| &ctx.nir->printf_info_count); | |
| } | |
| free(ctx.idx_table); | |
| _mesa_hash_table_fini(&ctx.strings, NULL); | |
| nir_validate_shader(ctx.nir, "after deserialize"); | |
| return ctx.nir; | |
| } | |
| nir_function * | |
| nir_deserialize_function(void *mem_ctx, | |
| const struct nir_shader_compiler_options *options, | |
| struct blob_reader *blob) | |
| { | |
| read_ctx ctx = { 0 }; | |
| ctx.blob = blob; | |
| list_inithead(&ctx.phi_srcs); | |
| ctx.idx_table_len = blob_read_uint32(blob); | |
| ctx.idx_table = calloc(ctx.idx_table_len, sizeof(uintptr_t)); | |
| ctx.nir = nir_shader_create(mem_ctx, 0 /* stage */, options); | |
| nir_function *fxn = read_function(&ctx); | |
| nir_function_set_impl(fxn, read_function_impl(&ctx)); | |
| free(ctx.idx_table); | |
| nir_validate_shader(ctx.nir, "after deserialize"); | |
| return fxn; | |
| } | |
| void | |
| nir_shader_serialize_deserialize(nir_shader *shader) | |
| { | |
| const struct nir_shader_compiler_options *options = shader->options; | |
| struct blob writer; | |
| blob_init(&writer); | |
| nir_serialize(&writer, shader, false); | |
| /* Delete all of dest's ralloc children but leave dest alone */ | |
| void *dead_ctx = ralloc_context(NULL); | |
| ralloc_adopt(dead_ctx, shader); | |
| ralloc_free(dead_ctx); | |
| dead_ctx = ralloc_context(NULL); | |
| struct blob_reader reader; | |
| blob_reader_init(&reader, writer.data, writer.size); | |
| nir_shader *copy = nir_deserialize(dead_ctx, options, &reader); | |
| blob_finish(&writer); | |
| nir_shader_replace(shader, copy); | |
| ralloc_free(dead_ctx); | |
| } | |
| #ifndef NDEBUG | |
| struct blob | |
| nir_validate_progress_setup(nir_shader *shader) | |
| { | |
| if (!NIR_DEBUG(PROGRESS_VALIDATION)) | |
| return (struct blob){0}; | |
| struct blob blob_before; | |
| blob_init(&blob_before); | |
| serialize_internal(&blob_before, shader, false, false); | |
| return blob_before; | |
| } | |
| void | |
| nir_validate_progress_finish(nir_shader *shader, struct blob *setup_blob, bool progress, const char *when) | |
| { | |
| if (!NIR_DEBUG(PROGRESS_VALIDATION)) | |
| return; | |
| if (!progress) { | |
| struct blob blob_after; | |
| blob_init(&blob_after); | |
| serialize_internal(&blob_after, shader, false, false); | |
| if (setup_blob->size != blob_after.size || | |
| memcmp(setup_blob->data, blob_after.data, setup_blob->size)) { | |
| fprintf(stderr, "NIR changed but no progress reported %s\n", when); | |
| abort(); | |
| } | |
| blob_finish(&blob_after); | |
| } | |
| blob_finish(setup_blob); | |
| } | |
| #endif |