| /* |
| * Copyright © 2014 Intel Corporation |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice (including the next |
| * paragraph) shall be included in all copies or substantial portions of the |
| * Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
| * IN THE SOFTWARE. |
| */ |
| |
| /** @file brw_fs_combine_constants.cpp |
| * |
| * This file contains the opt_combine_constants() pass that runs after the |
| * regular optimization loop. It passes over the instruction list and |
| * selectively promotes immediate values to registers by emitting a mov(1) |
| * instruction. |
| * |
| * This is useful on Gen 7 particularly, because a few instructions can be |
| * coissued (i.e., issued in the same cycle as another thread on the same EU |
| * issues an instruction) under some circumstances, one of which is that they |
| * cannot use immediate values. |
| */ |
| |
| #include "brw_fs.h" |
| #include "brw_cfg.h" |
| |
| using namespace brw; |
| |
| static const bool debug = false; |
| |
| /* Returns whether an instruction could co-issue if its immediate source were |
| * replaced with a GRF source. |
| */ |
| static bool |
| could_coissue(const struct gen_device_info *devinfo, const fs_inst *inst) |
| { |
| if (devinfo->gen != 7) |
| return false; |
| |
| switch (inst->opcode) { |
| case BRW_OPCODE_MOV: |
| case BRW_OPCODE_CMP: |
| case BRW_OPCODE_ADD: |
| case BRW_OPCODE_MUL: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| /** |
| * Returns true for instructions that don't support immediate sources. |
| */ |
| static bool |
| must_promote_imm(const struct gen_device_info *devinfo, const fs_inst *inst) |
| { |
| switch (inst->opcode) { |
| case SHADER_OPCODE_POW: |
| return devinfo->gen < 8; |
| case BRW_OPCODE_MAD: |
| case BRW_OPCODE_LRP: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| /** A box for putting fs_regs in a linked list. */ |
| struct reg_link { |
| DECLARE_RALLOC_CXX_OPERATORS(reg_link) |
| |
| reg_link(fs_reg *reg) : reg(reg) {} |
| |
| struct exec_node link; |
| fs_reg *reg; |
| }; |
| |
| static struct exec_node * |
| link(void *mem_ctx, fs_reg *reg) |
| { |
| reg_link *l = new(mem_ctx) reg_link(reg); |
| return &l->link; |
| } |
| |
| /** |
| * Information about an immediate value. |
| */ |
| struct imm { |
| /** The common ancestor of all blocks using this immediate value. */ |
| bblock_t *block; |
| |
| /** |
| * The instruction generating the immediate value, if all uses are contained |
| * within a single basic block. Otherwise, NULL. |
| */ |
| fs_inst *inst; |
| |
| /** |
| * A list of fs_regs that refer to this immediate. If we promote it, we'll |
| * have to patch these up to refer to the new GRF. |
| */ |
| exec_list *uses; |
| |
| /** The immediate value. We currently only handle floats. */ |
| float val; |
| |
| /** |
| * The GRF register and subregister number where we've decided to store the |
| * constant value. |
| */ |
| uint8_t subreg_offset; |
| uint16_t nr; |
| |
| /** The number of coissuable instructions using this immediate. */ |
| uint16_t uses_by_coissue; |
| |
| /** |
| * Whether this constant is used by an instruction that can't handle an |
| * immediate source (and already has to be promoted to a GRF). |
| */ |
| bool must_promote; |
| |
| uint16_t first_use_ip; |
| uint16_t last_use_ip; |
| }; |
| |
| /** The working set of information about immediates. */ |
| struct table { |
| struct imm *imm; |
| int size; |
| int len; |
| }; |
| |
| static struct imm * |
| find_imm(struct table *table, float val) |
| { |
| for (int i = 0; i < table->len; i++) { |
| if (table->imm[i].val == val) { |
| return &table->imm[i]; |
| } |
| } |
| return NULL; |
| } |
| |
| static struct imm * |
| new_imm(struct table *table, void *mem_ctx) |
| { |
| if (table->len == table->size) { |
| table->size *= 2; |
| table->imm = reralloc(mem_ctx, table->imm, struct imm, table->size); |
| } |
| return &table->imm[table->len++]; |
| } |
| |
| /** |
| * Comparator used for sorting an array of imm structures. |
| * |
| * We sort by basic block number, then last use IP, then first use IP (least |
| * to greatest). This sorting causes immediates live in the same area to be |
| * allocated to the same register in the hopes that all values will be dead |
| * about the same time and the register can be reused. |
| */ |
| static int |
| compare(const void *_a, const void *_b) |
| { |
| const struct imm *a = (const struct imm *)_a, |
| *b = (const struct imm *)_b; |
| |
| int block_diff = a->block->num - b->block->num; |
| if (block_diff) |
| return block_diff; |
| |
| int end_diff = a->last_use_ip - b->last_use_ip; |
| if (end_diff) |
| return end_diff; |
| |
| return a->first_use_ip - b->first_use_ip; |
| } |
| |
| bool |
| fs_visitor::opt_combine_constants() |
| { |
| void *const_ctx = ralloc_context(NULL); |
| |
| struct table table; |
| table.size = 8; |
| table.len = 0; |
| table.imm = ralloc_array(const_ctx, struct imm, table.size); |
| |
| cfg->calculate_idom(); |
| unsigned ip = -1; |
| |
| /* Make a pass through all instructions and count the number of times each |
| * constant is used by coissueable instructions or instructions that cannot |
| * take immediate arguments. |
| */ |
| foreach_block_and_inst(block, fs_inst, inst, cfg) { |
| ip++; |
| |
| if (!could_coissue(devinfo, inst) && !must_promote_imm(devinfo, inst)) |
| continue; |
| |
| for (int i = 0; i < inst->sources; i++) { |
| if (inst->src[i].file != IMM || |
| inst->src[i].type != BRW_REGISTER_TYPE_F) |
| continue; |
| |
| float val = !inst->can_do_source_mods(devinfo) ? inst->src[i].f : |
| fabs(inst->src[i].f); |
| struct imm *imm = find_imm(&table, val); |
| |
| if (imm) { |
| bblock_t *intersection = cfg_t::intersect(block, imm->block); |
| if (intersection != imm->block) |
| imm->inst = NULL; |
| imm->block = intersection; |
| imm->uses->push_tail(link(const_ctx, &inst->src[i])); |
| imm->uses_by_coissue += could_coissue(devinfo, inst); |
| imm->must_promote = imm->must_promote || must_promote_imm(devinfo, inst); |
| imm->last_use_ip = ip; |
| } else { |
| imm = new_imm(&table, const_ctx); |
| imm->block = block; |
| imm->inst = inst; |
| imm->uses = new(const_ctx) exec_list(); |
| imm->uses->push_tail(link(const_ctx, &inst->src[i])); |
| imm->val = val; |
| imm->uses_by_coissue = could_coissue(devinfo, inst); |
| imm->must_promote = must_promote_imm(devinfo, inst); |
| imm->first_use_ip = ip; |
| imm->last_use_ip = ip; |
| } |
| } |
| } |
| |
| /* Remove constants from the table that don't have enough uses to make them |
| * profitable to store in a register. |
| */ |
| for (int i = 0; i < table.len;) { |
| struct imm *imm = &table.imm[i]; |
| |
| if (!imm->must_promote && imm->uses_by_coissue < 4) { |
| table.imm[i] = table.imm[table.len - 1]; |
| table.len--; |
| continue; |
| } |
| i++; |
| } |
| if (table.len == 0) { |
| ralloc_free(const_ctx); |
| return false; |
| } |
| if (cfg->num_blocks != 1) |
| qsort(table.imm, table.len, sizeof(struct imm), compare); |
| |
| /* Insert MOVs to load the constant values into GRFs. */ |
| fs_reg reg(VGRF, alloc.allocate(1)); |
| reg.stride = 0; |
| for (int i = 0; i < table.len; i++) { |
| struct imm *imm = &table.imm[i]; |
| /* Insert it either before the instruction that generated the immediate |
| * or after the last non-control flow instruction of the common ancestor. |
| */ |
| exec_node *n = (imm->inst ? imm->inst : |
| imm->block->last_non_control_flow_inst()->next); |
| const fs_builder ibld = bld.at(imm->block, n).exec_all().group(1, 0); |
| |
| ibld.MOV(reg, brw_imm_f(imm->val)); |
| imm->nr = reg.nr; |
| imm->subreg_offset = reg.offset; |
| |
| reg.offset += sizeof(float); |
| if (reg.offset == 8 * sizeof(float)) { |
| reg.nr = alloc.allocate(1); |
| reg.offset = 0; |
| } |
| } |
| promoted_constants = table.len; |
| |
| /* Rewrite the immediate sources to refer to the new GRFs. */ |
| for (int i = 0; i < table.len; i++) { |
| foreach_list_typed(reg_link, link, link, table.imm[i].uses) { |
| fs_reg *reg = link->reg; |
| reg->file = VGRF; |
| reg->nr = table.imm[i].nr; |
| reg->offset = table.imm[i].subreg_offset; |
| reg->stride = 0; |
| reg->negate = signbit(reg->f) != signbit(table.imm[i].val); |
| assert((isnan(reg->f) && isnan(table.imm[i].val)) || |
| fabsf(reg->f) == fabs(table.imm[i].val)); |
| } |
| } |
| |
| if (debug) { |
| for (int i = 0; i < table.len; i++) { |
| struct imm *imm = &table.imm[i]; |
| |
| printf("%.3fF - block %3d, reg %3d sub %2d, Uses: (%2d, %2d), " |
| "IP: %4d to %4d, length %4d\n", |
| imm->val, |
| imm->block->num, |
| imm->nr, |
| imm->subreg_offset, |
| imm->must_promote, |
| imm->uses_by_coissue, |
| imm->first_use_ip, |
| imm->last_use_ip, |
| imm->last_use_ip - imm->first_use_ip); |
| } |
| } |
| |
| ralloc_free(const_ctx); |
| invalidate_live_intervals(); |
| |
| return true; |
| } |