| /* |
| * Copyright (C) 2016 Rob Clark <robclark@freedesktop.org> |
| * Copyright © 2018 Google, Inc. |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice (including the next |
| * paragraph) shall be included in all copies or substantial portions of the |
| * Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| * SOFTWARE. |
| * |
| * Authors: |
| * Rob Clark <robclark@freedesktop.org> |
| */ |
| |
| #include "pipe/p_state.h" |
| #include "util/u_string.h" |
| #include "util/u_memory.h" |
| #include "util/u_prim.h" |
| |
| #include "freedreno_state.h" |
| #include "freedreno_resource.h" |
| |
| #include "fd6_draw.h" |
| #include "fd6_context.h" |
| #include "fd6_emit.h" |
| #include "fd6_program.h" |
| #include "fd6_format.h" |
| #include "fd6_vsc.h" |
| #include "fd6_zsa.h" |
| |
| #include "fd6_pack.h" |
| |
| static void |
| draw_emit_indirect(struct fd_ringbuffer *ring, |
| struct CP_DRAW_INDX_OFFSET_0 *draw0, |
| const struct pipe_draw_info *info, |
| unsigned index_offset) |
| { |
| struct fd_resource *ind = fd_resource(info->indirect->buffer); |
| |
| if (info->index_size) { |
| struct pipe_resource *idx = info->index.resource; |
| unsigned max_indices = (idx->width0 - index_offset) / info->index_size; |
| |
| OUT_PKT(ring, CP_DRAW_INDX_INDIRECT, |
| pack_CP_DRAW_INDX_OFFSET_0(*draw0), |
| A5XX_CP_DRAW_INDX_INDIRECT_INDX_BASE( |
| fd_resource(idx)->bo, index_offset), |
| A5XX_CP_DRAW_INDX_INDIRECT_3(.max_indices = max_indices), |
| A5XX_CP_DRAW_INDX_INDIRECT_INDIRECT( |
| ind->bo, info->indirect->offset) |
| ); |
| } else { |
| OUT_PKT(ring, CP_DRAW_INDIRECT, |
| pack_CP_DRAW_INDX_OFFSET_0(*draw0), |
| A5XX_CP_DRAW_INDIRECT_INDIRECT( |
| ind->bo, info->indirect->offset) |
| ); |
| } |
| } |
| |
| static void |
| draw_emit(struct fd_ringbuffer *ring, |
| struct CP_DRAW_INDX_OFFSET_0 *draw0, |
| const struct pipe_draw_info *info, |
| unsigned index_offset) |
| { |
| if (info->index_size) { |
| assert(!info->has_user_indices); |
| |
| struct pipe_resource *idx_buffer = info->index.resource; |
| unsigned max_indices = (idx_buffer->width0 - index_offset) / info->index_size; |
| |
| OUT_PKT(ring, CP_DRAW_INDX_OFFSET, |
| pack_CP_DRAW_INDX_OFFSET_0(*draw0), |
| CP_DRAW_INDX_OFFSET_1(.num_instances = info->instance_count), |
| CP_DRAW_INDX_OFFSET_2(.num_indices = info->count), |
| CP_DRAW_INDX_OFFSET_3(.first_indx = info->start), |
| A5XX_CP_DRAW_INDX_OFFSET_INDX_BASE( |
| fd_resource(idx_buffer)->bo, index_offset), |
| A5XX_CP_DRAW_INDX_OFFSET_6(.max_indices = max_indices) |
| ); |
| } else { |
| OUT_PKT(ring, CP_DRAW_INDX_OFFSET, |
| pack_CP_DRAW_INDX_OFFSET_0(*draw0), |
| CP_DRAW_INDX_OFFSET_1(.num_instances = info->instance_count), |
| CP_DRAW_INDX_OFFSET_2(.num_indices = info->count) |
| ); |
| } |
| } |
| |
| /* fixup dirty shader state in case some "unrelated" (from the state- |
| * tracker's perspective) state change causes us to switch to a |
| * different variant. |
| */ |
| static void |
| fixup_shader_state(struct fd_context *ctx, struct ir3_shader_key *key) |
| { |
| struct fd6_context *fd6_ctx = fd6_context(ctx); |
| struct ir3_shader_key *last_key = &fd6_ctx->last_key; |
| |
| if (!ir3_shader_key_equal(last_key, key)) { |
| if (ir3_shader_key_changes_fs(last_key, key)) { |
| ctx->dirty_shader[PIPE_SHADER_FRAGMENT] |= FD_DIRTY_SHADER_PROG; |
| ctx->dirty |= FD_DIRTY_PROG; |
| } |
| |
| if (ir3_shader_key_changes_vs(last_key, key)) { |
| ctx->dirty_shader[PIPE_SHADER_VERTEX] |= FD_DIRTY_SHADER_PROG; |
| ctx->dirty |= FD_DIRTY_PROG; |
| } |
| |
| fd6_ctx->last_key = *key; |
| } |
| } |
| |
| static void |
| fixup_draw_state(struct fd_context *ctx, struct fd6_emit *emit) |
| { |
| if (ctx->last.dirty || |
| (ctx->last.primitive_restart != emit->primitive_restart)) { |
| /* rasterizer state is effected by primitive-restart: */ |
| ctx->dirty |= FD_DIRTY_RASTERIZER; |
| ctx->last.primitive_restart = emit->primitive_restart; |
| } |
| } |
| |
| static bool |
| fd6_draw_vbo(struct fd_context *ctx, const struct pipe_draw_info *info, |
| unsigned index_offset) |
| { |
| struct fd6_context *fd6_ctx = fd6_context(ctx); |
| struct ir3_shader *gs = ctx->prog.gs; |
| struct fd6_emit emit = { |
| .ctx = ctx, |
| .vtx = &ctx->vtx, |
| .info = info, |
| .key = { |
| .vs = ctx->prog.vs, |
| .gs = ctx->prog.gs, |
| .fs = ctx->prog.fs, |
| .key = { |
| .color_two_side = ctx->rasterizer->light_twoside, |
| .vclamp_color = ctx->rasterizer->clamp_vertex_color, |
| .fclamp_color = ctx->rasterizer->clamp_fragment_color, |
| .rasterflat = ctx->rasterizer->flatshade, |
| .ucp_enables = ctx->rasterizer->clip_plane_enable, |
| .has_per_samp = (fd6_ctx->fsaturate || fd6_ctx->vsaturate), |
| .vsaturate_s = fd6_ctx->vsaturate_s, |
| .vsaturate_t = fd6_ctx->vsaturate_t, |
| .vsaturate_r = fd6_ctx->vsaturate_r, |
| .fsaturate_s = fd6_ctx->fsaturate_s, |
| .fsaturate_t = fd6_ctx->fsaturate_t, |
| .fsaturate_r = fd6_ctx->fsaturate_r, |
| .layer_zero = !gs || !(gs->nir->info.outputs_written & VARYING_BIT_LAYER), |
| .vsamples = ctx->tex[PIPE_SHADER_VERTEX].samples, |
| .fsamples = ctx->tex[PIPE_SHADER_FRAGMENT].samples, |
| .sample_shading = (ctx->min_samples > 1), |
| .msaa = (ctx->framebuffer.samples > 1), |
| }, |
| }, |
| .rasterflat = ctx->rasterizer->flatshade, |
| .sprite_coord_enable = ctx->rasterizer->sprite_coord_enable, |
| .sprite_coord_mode = ctx->rasterizer->sprite_coord_mode, |
| .primitive_restart = info->primitive_restart && info->index_size, |
| }; |
| |
| if (info->mode == PIPE_PRIM_PATCHES) { |
| emit.key.hs = ctx->prog.hs; |
| emit.key.ds = ctx->prog.ds; |
| |
| shader_info *ds_info = &emit.key.ds->nir->info; |
| emit.key.key.tessellation = ir3_tess_mode(ds_info->tess.primitive_mode); |
| } |
| |
| if (emit.key.gs) |
| emit.key.key.has_gs = true; |
| |
| if (!(emit.key.hs || emit.key.ds || emit.key.gs || info->indirect)) |
| fd6_vsc_update_sizes(ctx->batch, info); |
| |
| fixup_shader_state(ctx, &emit.key.key); |
| |
| if (!(ctx->dirty & FD_DIRTY_PROG)) { |
| emit.prog = fd6_ctx->prog; |
| } else { |
| fd6_ctx->prog = fd6_emit_get_prog(&emit); |
| } |
| |
| /* bail if compile failed: */ |
| if (!fd6_ctx->prog) |
| return NULL; |
| |
| emit.dirty = ctx->dirty; /* *after* fixup_shader_state() */ |
| emit.bs = fd6_emit_get_prog(&emit)->bs; |
| emit.vs = fd6_emit_get_prog(&emit)->vs; |
| emit.hs = fd6_emit_get_prog(&emit)->hs; |
| emit.ds = fd6_emit_get_prog(&emit)->ds; |
| emit.gs = fd6_emit_get_prog(&emit)->gs; |
| emit.fs = fd6_emit_get_prog(&emit)->fs; |
| |
| ctx->stats.vs_regs += ir3_shader_halfregs(emit.vs); |
| ctx->stats.hs_regs += COND(emit.hs, ir3_shader_halfregs(emit.hs)); |
| ctx->stats.ds_regs += COND(emit.ds, ir3_shader_halfregs(emit.ds)); |
| ctx->stats.gs_regs += COND(emit.gs, ir3_shader_halfregs(emit.gs)); |
| ctx->stats.fs_regs += ir3_shader_halfregs(emit.fs); |
| |
| struct fd_ringbuffer *ring = ctx->batch->draw; |
| |
| struct CP_DRAW_INDX_OFFSET_0 draw0 = { |
| .prim_type = ctx->primtypes[info->mode], |
| .vis_cull = USE_VISIBILITY, |
| .gs_enable = !!emit.key.gs, |
| }; |
| |
| if (info->index_size) { |
| draw0.source_select = DI_SRC_SEL_DMA; |
| draw0.index_size = fd4_size2indextype(info->index_size); |
| } else { |
| draw0.source_select = DI_SRC_SEL_AUTO_INDEX; |
| } |
| |
| if (info->mode == PIPE_PRIM_PATCHES) { |
| shader_info *ds_info = &emit.ds->shader->nir->info; |
| uint32_t factor_stride; |
| |
| switch (ds_info->tess.primitive_mode) { |
| case GL_ISOLINES: |
| draw0.patch_type = TESS_ISOLINES; |
| factor_stride = 12; |
| break; |
| case GL_TRIANGLES: |
| draw0.patch_type = TESS_TRIANGLES; |
| factor_stride = 20; |
| break; |
| case GL_QUADS: |
| draw0.patch_type = TESS_QUADS; |
| factor_stride = 28; |
| break; |
| default: |
| unreachable("bad tessmode"); |
| } |
| |
| draw0.prim_type = DI_PT_PATCHES0 + info->vertices_per_patch; |
| draw0.tess_enable = true; |
| |
| ctx->batch->tessellation = true; |
| ctx->batch->tessparam_size = MAX2(ctx->batch->tessparam_size, |
| emit.hs->output_size * 4 * info->count); |
| ctx->batch->tessfactor_size = MAX2(ctx->batch->tessfactor_size, |
| factor_stride * info->count); |
| |
| if (!ctx->batch->tess_addrs_constobj) { |
| /* Reserve space for the bo address - we'll write them later in |
| * setup_tess_buffers(). We need 2 bo address, but indirect |
| * constant upload needs at least 4 vec4s. |
| */ |
| unsigned size = 4 * 16; |
| |
| ctx->batch->tess_addrs_constobj = fd_submit_new_ringbuffer( |
| ctx->batch->submit, size, FD_RINGBUFFER_STREAMING); |
| |
| ctx->batch->tess_addrs_constobj->cur += size; |
| } |
| } |
| |
| uint32_t index_start = info->index_size ? info->index_bias : info->start; |
| if (ctx->last.dirty || (ctx->last.index_start != index_start)) { |
| OUT_PKT4(ring, REG_A6XX_VFD_INDEX_OFFSET, 1); |
| OUT_RING(ring, index_start); /* VFD_INDEX_OFFSET */ |
| ctx->last.index_start = index_start; |
| } |
| |
| if (ctx->last.dirty || (ctx->last.instance_start != info->start_instance)) { |
| OUT_PKT4(ring, REG_A6XX_VFD_INSTANCE_START_OFFSET, 1); |
| OUT_RING(ring, info->start_instance); /* VFD_INSTANCE_START_OFFSET */ |
| ctx->last.instance_start = info->start_instance; |
| } |
| |
| uint32_t restart_index = info->primitive_restart ? info->restart_index : 0xffffffff; |
| if (ctx->last.dirty || (ctx->last.restart_index != restart_index)) { |
| OUT_PKT4(ring, REG_A6XX_PC_RESTART_INDEX, 1); |
| OUT_RING(ring, restart_index); /* PC_RESTART_INDEX */ |
| ctx->last.restart_index = restart_index; |
| } |
| |
| fixup_draw_state(ctx, &emit); |
| |
| fd6_emit_state(ring, &emit); |
| |
| /* for debug after a lock up, write a unique counter value |
| * to scratch7 for each draw, to make it easier to match up |
| * register dumps to cmdstream. The combination of IB |
| * (scratch6) and DRAW is enough to "triangulate" the |
| * particular draw that caused lockup. |
| */ |
| emit_marker6(ring, 7); |
| |
| if (info->indirect) { |
| draw_emit_indirect(ring, &draw0, info, index_offset); |
| } else { |
| draw_emit(ring, &draw0, info, index_offset); |
| } |
| |
| emit_marker6(ring, 7); |
| fd_reset_wfi(ctx->batch); |
| |
| if (emit.streamout_mask) { |
| struct fd_ringbuffer *ring = ctx->batch->draw; |
| |
| for (unsigned i = 0; i < PIPE_MAX_SO_BUFFERS; i++) { |
| if (emit.streamout_mask & (1 << i)) { |
| fd6_event_write(ctx->batch, ring, FLUSH_SO_0 + i, false); |
| } |
| } |
| } |
| |
| fd_context_all_clean(ctx); |
| |
| return true; |
| } |
| |
| static void |
| fd6_clear_lrz(struct fd_batch *batch, struct fd_resource *zsbuf, double depth) |
| { |
| struct fd_ringbuffer *ring; |
| struct fd6_context *fd6_ctx = fd6_context(batch->ctx); |
| |
| if (batch->lrz_clear) { |
| fd_ringbuffer_del(batch->lrz_clear); |
| } |
| |
| batch->lrz_clear = fd_submit_new_ringbuffer(batch->submit, 0x1000, 0); |
| ring = batch->lrz_clear; |
| |
| emit_marker6(ring, 7); |
| OUT_PKT7(ring, CP_SET_MARKER, 1); |
| OUT_RING(ring, A6XX_CP_SET_MARKER_0_MODE(RM6_BYPASS)); |
| emit_marker6(ring, 7); |
| |
| OUT_WFI5(ring); |
| |
| OUT_PKT4(ring, REG_A6XX_RB_CCU_CNTL, 1); |
| OUT_RING(ring, fd6_ctx->magic.RB_CCU_CNTL_bypass); |
| |
| OUT_REG(ring, A6XX_HLSQ_INVALIDATE_CMD( |
| .vs_state = true, |
| .hs_state = true, |
| .ds_state = true, |
| .gs_state = true, |
| .fs_state = true, |
| .cs_state = true, |
| .gfx_ibo = true, |
| .cs_ibo = true, |
| .gfx_shared_const = true, |
| .gfx_bindless = 0x1f, |
| .cs_bindless = 0x1f |
| )); |
| |
| emit_marker6(ring, 7); |
| OUT_PKT7(ring, CP_SET_MARKER, 1); |
| OUT_RING(ring, A6XX_CP_SET_MARKER_0_MODE(RM6_BLIT2DSCALE)); |
| emit_marker6(ring, 7); |
| |
| OUT_PKT4(ring, REG_A6XX_RB_UNKNOWN_8C01, 1); |
| OUT_RING(ring, 0x0); |
| |
| OUT_PKT4(ring, REG_A6XX_SP_PS_2D_SRC_INFO, 13); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| |
| OUT_PKT4(ring, REG_A6XX_SP_2D_DST_FORMAT, 1); |
| OUT_RING(ring, 0x0000f410); |
| |
| OUT_PKT4(ring, REG_A6XX_GRAS_2D_BLIT_CNTL, 1); |
| OUT_RING(ring, A6XX_GRAS_2D_BLIT_CNTL_COLOR_FORMAT(FMT6_16_UNORM) | |
| 0x4f00080); |
| |
| OUT_PKT4(ring, REG_A6XX_RB_2D_BLIT_CNTL, 1); |
| OUT_RING(ring, A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT(FMT6_16_UNORM) | |
| 0x4f00080); |
| |
| fd6_event_write(batch, ring, PC_CCU_FLUSH_COLOR_TS, true); |
| fd6_event_write(batch, ring, PC_CCU_INVALIDATE_COLOR, false); |
| |
| OUT_PKT4(ring, REG_A6XX_RB_2D_SRC_SOLID_C0, 4); |
| OUT_RING(ring, fui(depth)); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| |
| OUT_PKT4(ring, REG_A6XX_RB_2D_DST_INFO, 9); |
| OUT_RING(ring, A6XX_RB_2D_DST_INFO_COLOR_FORMAT(FMT6_16_UNORM) | |
| A6XX_RB_2D_DST_INFO_TILE_MODE(TILE6_LINEAR) | |
| A6XX_RB_2D_DST_INFO_COLOR_SWAP(WZYX)); |
| OUT_RELOC(ring, zsbuf->lrz, 0, 0, 0); |
| OUT_RING(ring, A6XX_RB_2D_DST_SIZE_PITCH(zsbuf->lrz_pitch * 2)); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| OUT_RING(ring, 0x00000000); |
| |
| OUT_PKT4(ring, REG_A6XX_GRAS_2D_SRC_TL_X, 4); |
| OUT_RING(ring, A6XX_GRAS_2D_SRC_TL_X_X(0)); |
| OUT_RING(ring, A6XX_GRAS_2D_SRC_BR_X_X(0)); |
| OUT_RING(ring, A6XX_GRAS_2D_SRC_TL_Y_Y(0)); |
| OUT_RING(ring, A6XX_GRAS_2D_SRC_BR_Y_Y(0)); |
| |
| OUT_PKT4(ring, REG_A6XX_GRAS_2D_DST_TL, 2); |
| OUT_RING(ring, A6XX_GRAS_2D_DST_TL_X(0) | |
| A6XX_GRAS_2D_DST_TL_Y(0)); |
| OUT_RING(ring, A6XX_GRAS_2D_DST_BR_X(zsbuf->lrz_width - 1) | |
| A6XX_GRAS_2D_DST_BR_Y(zsbuf->lrz_height - 1)); |
| |
| fd6_event_write(batch, ring, 0x3f, false); |
| |
| OUT_WFI5(ring); |
| |
| OUT_PKT4(ring, REG_A6XX_RB_UNKNOWN_8E04, 1); |
| OUT_RING(ring, fd6_ctx->magic.RB_UNKNOWN_8E04_blit); |
| |
| OUT_PKT7(ring, CP_BLIT, 1); |
| OUT_RING(ring, CP_BLIT_0_OP(BLIT_OP_SCALE)); |
| |
| OUT_WFI5(ring); |
| |
| OUT_PKT4(ring, REG_A6XX_RB_UNKNOWN_8E04, 1); |
| OUT_RING(ring, 0x0); /* RB_UNKNOWN_8E04 */ |
| |
| fd6_event_write(batch, ring, PC_CCU_FLUSH_COLOR_TS, true); |
| fd6_event_write(batch, ring, PC_CCU_FLUSH_DEPTH_TS, true); |
| fd6_event_write(batch, ring, CACHE_FLUSH_TS, true); |
| |
| fd6_cache_inv(batch, ring); |
| } |
| |
| static bool is_z32(enum pipe_format format) |
| { |
| switch (format) { |
| case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: |
| case PIPE_FORMAT_Z32_UNORM: |
| case PIPE_FORMAT_Z32_FLOAT: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| static bool |
| fd6_clear(struct fd_context *ctx, unsigned buffers, |
| const union pipe_color_union *color, double depth, unsigned stencil) |
| { |
| struct pipe_framebuffer_state *pfb = &ctx->batch->framebuffer; |
| const bool has_depth = pfb->zsbuf; |
| unsigned color_buffers = buffers >> 2; |
| |
| /* If we're clearing after draws, fallback to 3D pipe clears. We could |
| * use blitter clears in the draw batch but then we'd have to patch up the |
| * gmem offsets. This doesn't seem like a useful thing to optimize for |
| * however.*/ |
| if (ctx->batch->num_draws > 0) |
| return false; |
| |
| foreach_bit(i, color_buffers) |
| ctx->batch->clear_color[i] = *color; |
| if (buffers & PIPE_CLEAR_DEPTH) |
| ctx->batch->clear_depth = depth; |
| if (buffers & PIPE_CLEAR_STENCIL) |
| ctx->batch->clear_stencil = stencil; |
| |
| ctx->batch->fast_cleared |= buffers; |
| |
| if (has_depth && (buffers & PIPE_CLEAR_DEPTH)) { |
| struct fd_resource *zsbuf = fd_resource(pfb->zsbuf->texture); |
| if (zsbuf->lrz && !is_z32(pfb->zsbuf->format)) { |
| zsbuf->lrz_valid = true; |
| zsbuf->lrz_direction = FD_LRZ_UNKNOWN; |
| fd6_clear_lrz(ctx->batch, zsbuf, depth); |
| } |
| } |
| |
| return true; |
| } |
| |
| void |
| fd6_draw_init(struct pipe_context *pctx) |
| { |
| struct fd_context *ctx = fd_context(pctx); |
| ctx->draw_vbo = fd6_draw_vbo; |
| ctx->clear = fd6_clear; |
| } |