| /* |
| * Copyright © 2019 Raspberry Pi |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice (including the next |
| * paragraph) shall be included in all copies or substantial portions of the |
| * Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
| * IN THE SOFTWARE. |
| */ |
| |
| #include "v3dv_private.h" |
| #include "broadcom/cle/v3dx_pack.h" |
| #include "util/half_float.h" |
| #include "util/u_pack_color.h" |
| #include "vk_format_info.h" |
| |
| const struct v3dv_dynamic_state default_dynamic_state = { |
| .viewport = { |
| .count = 0, |
| }, |
| .scissor = { |
| .count = 0, |
| }, |
| .stencil_compare_mask = |
| { |
| .front = ~0u, |
| .back = ~0u, |
| }, |
| .stencil_write_mask = |
| { |
| .front = ~0u, |
| .back = ~0u, |
| }, |
| .stencil_reference = |
| { |
| .front = 0u, |
| .back = 0u, |
| }, |
| .blend_constants = { 0.0f, 0.0f, 0.0f, 0.0f }, |
| .depth_bias = { |
| .constant_factor = 0.0f, |
| .slope_factor = 0.0f, |
| }, |
| .line_width = 1.0f, |
| }; |
| |
| void |
| v3dv_job_add_bo(struct v3dv_job *job, struct v3dv_bo *bo) |
| { |
| if (!bo) |
| return; |
| |
| if (_mesa_set_search(job->bos, bo)) |
| return; |
| |
| _mesa_set_add(job->bos, bo); |
| job->bo_count++; |
| } |
| |
| static void |
| cmd_buffer_emit_render_pass_rcl(struct v3dv_cmd_buffer *cmd_buffer); |
| |
| VkResult |
| v3dv_CreateCommandPool(VkDevice _device, |
| const VkCommandPoolCreateInfo *pCreateInfo, |
| const VkAllocationCallbacks *pAllocator, |
| VkCommandPool *pCmdPool) |
| { |
| V3DV_FROM_HANDLE(v3dv_device, device, _device); |
| struct v3dv_cmd_pool *pool; |
| |
| /* We only support one queue */ |
| assert(pCreateInfo->queueFamilyIndex == 0); |
| |
| pool = vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); |
| if (pool == NULL) |
| return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| if (pAllocator) |
| pool->alloc = *pAllocator; |
| else |
| pool->alloc = device->alloc; |
| |
| list_inithead(&pool->cmd_buffers); |
| |
| *pCmdPool = v3dv_cmd_pool_to_handle(pool); |
| |
| return VK_SUCCESS; |
| } |
| |
| static void |
| cmd_buffer_init(struct v3dv_cmd_buffer *cmd_buffer, |
| struct v3dv_device *device, |
| struct v3dv_cmd_pool *pool, |
| VkCommandBufferLevel level) |
| { |
| /* Do not reset the loader data header! If we are calling this from |
| * a command buffer reset that would reset the loader's dispatch table for |
| * the command buffer. |
| */ |
| const uint32_t ld_size = sizeof(VK_LOADER_DATA); |
| uint8_t *cmd_buffer_driver_start = ((uint8_t *) cmd_buffer) + ld_size; |
| memset(cmd_buffer_driver_start, 0, sizeof(*cmd_buffer) - ld_size); |
| |
| cmd_buffer->device = device; |
| cmd_buffer->pool = pool; |
| cmd_buffer->level = level; |
| |
| list_inithead(&cmd_buffer->private_objs); |
| list_inithead(&cmd_buffer->jobs); |
| list_inithead(&cmd_buffer->list_link); |
| |
| assert(pool); |
| list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers); |
| |
| cmd_buffer->state.subpass_idx = -1; |
| cmd_buffer->state.meta.subpass_idx = -1; |
| |
| cmd_buffer->status = V3DV_CMD_BUFFER_STATUS_INITIALIZED; |
| } |
| |
| static VkResult |
| cmd_buffer_create(struct v3dv_device *device, |
| struct v3dv_cmd_pool *pool, |
| VkCommandBufferLevel level, |
| VkCommandBuffer *pCommandBuffer) |
| { |
| struct v3dv_cmd_buffer *cmd_buffer; |
| cmd_buffer = vk_alloc(&pool->alloc, sizeof(*cmd_buffer), 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); |
| if (cmd_buffer == NULL) |
| return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| cmd_buffer_init(cmd_buffer, device, pool, level); |
| |
| cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC; |
| |
| *pCommandBuffer = v3dv_cmd_buffer_to_handle(cmd_buffer); |
| |
| return VK_SUCCESS; |
| } |
| |
| static void |
| job_destroy_gpu_cl_resources(struct v3dv_job *job) |
| { |
| assert(job->type == V3DV_JOB_TYPE_GPU_CL || |
| job->type == V3DV_JOB_TYPE_GPU_CL_SECONDARY); |
| |
| v3dv_cl_destroy(&job->bcl); |
| v3dv_cl_destroy(&job->rcl); |
| v3dv_cl_destroy(&job->indirect); |
| |
| /* Since we don't ref BOs when we add them to the command buffer, don't |
| * unref them here either. Bo's will be freed when their corresponding API |
| * objects are destroyed. |
| */ |
| _mesa_set_destroy(job->bos, NULL); |
| |
| v3dv_bo_free(job->device, job->tile_alloc); |
| v3dv_bo_free(job->device, job->tile_state); |
| } |
| |
| static void |
| job_destroy_cloned_gpu_cl_resources(struct v3dv_job *job) |
| { |
| assert(job->type == V3DV_JOB_TYPE_GPU_CL); |
| |
| list_for_each_entry_safe(struct v3dv_bo, bo, &job->bcl.bo_list, list_link) { |
| list_del(&bo->list_link); |
| vk_free(&job->device->alloc, bo); |
| } |
| |
| list_for_each_entry_safe(struct v3dv_bo, bo, &job->rcl.bo_list, list_link) { |
| list_del(&bo->list_link); |
| vk_free(&job->device->alloc, bo); |
| } |
| |
| list_for_each_entry_safe(struct v3dv_bo, bo, &job->indirect.bo_list, list_link) { |
| list_del(&bo->list_link); |
| vk_free(&job->device->alloc, bo); |
| } |
| } |
| |
| static void |
| job_destroy_gpu_csd_resources(struct v3dv_job *job) |
| { |
| assert(job->type == V3DV_JOB_TYPE_GPU_CSD); |
| assert(job->cmd_buffer); |
| |
| v3dv_cl_destroy(&job->indirect); |
| |
| _mesa_set_destroy(job->bos, NULL); |
| |
| if (job->csd.shared_memory) |
| v3dv_bo_free(job->device, job->csd.shared_memory); |
| } |
| |
| static void |
| job_destroy_cpu_wait_events_resources(struct v3dv_job *job) |
| { |
| assert(job->type == V3DV_JOB_TYPE_CPU_WAIT_EVENTS); |
| assert(job->cmd_buffer); |
| vk_free(&job->cmd_buffer->device->alloc, job->cpu.event_wait.events); |
| } |
| |
| static void |
| job_destroy_cpu_csd_indirect_resources(struct v3dv_job *job) |
| { |
| assert(job->type == V3DV_JOB_TYPE_CPU_CSD_INDIRECT); |
| assert(job->cmd_buffer); |
| v3dv_job_destroy(job->cpu.csd_indirect.csd_job); |
| } |
| |
| void |
| v3dv_job_destroy(struct v3dv_job *job) |
| { |
| assert(job); |
| |
| list_del(&job->list_link); |
| |
| /* Cloned jobs don't make deep copies of the original jobs, so they don't |
| * own any of their resources. However, they do allocate clones of BO |
| * structs, so make sure we free those. |
| */ |
| if (!job->is_clone) { |
| switch (job->type) { |
| case V3DV_JOB_TYPE_GPU_CL: |
| case V3DV_JOB_TYPE_GPU_CL_SECONDARY: |
| job_destroy_gpu_cl_resources(job); |
| break; |
| case V3DV_JOB_TYPE_GPU_CSD: |
| job_destroy_gpu_csd_resources(job); |
| break; |
| case V3DV_JOB_TYPE_CPU_WAIT_EVENTS: |
| job_destroy_cpu_wait_events_resources(job); |
| break; |
| case V3DV_JOB_TYPE_CPU_CSD_INDIRECT: |
| job_destroy_cpu_csd_indirect_resources(job); |
| break; |
| default: |
| break; |
| } |
| } else { |
| /* Cloned jobs */ |
| if (job->type == V3DV_JOB_TYPE_GPU_CL) |
| job_destroy_cloned_gpu_cl_resources(job); |
| } |
| |
| vk_free(&job->device->alloc, job); |
| } |
| |
| void |
| v3dv_cmd_buffer_add_private_obj(struct v3dv_cmd_buffer *cmd_buffer, |
| uint64_t obj, |
| v3dv_cmd_buffer_private_obj_destroy_cb destroy_cb) |
| { |
| struct v3dv_cmd_buffer_private_obj *pobj = |
| vk_alloc(&cmd_buffer->device->alloc, sizeof(*pobj), 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); |
| if (!pobj) { |
| v3dv_flag_oom(cmd_buffer, NULL); |
| return; |
| } |
| |
| pobj->obj = obj; |
| pobj->destroy_cb = destroy_cb; |
| |
| list_addtail(&pobj->list_link, &cmd_buffer->private_objs); |
| } |
| |
| static void |
| cmd_buffer_destroy_private_obj(struct v3dv_cmd_buffer *cmd_buffer, |
| struct v3dv_cmd_buffer_private_obj *pobj) |
| { |
| assert(pobj && pobj->obj && pobj->destroy_cb); |
| pobj->destroy_cb(v3dv_device_to_handle(cmd_buffer->device), |
| pobj->obj, |
| &cmd_buffer->device->alloc); |
| list_del(&pobj->list_link); |
| vk_free(&cmd_buffer->device->alloc, pobj); |
| } |
| |
| static void |
| cmd_buffer_free_resources(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| list_for_each_entry_safe(struct v3dv_job, job, |
| &cmd_buffer->jobs, list_link) { |
| v3dv_job_destroy(job); |
| } |
| |
| if (cmd_buffer->state.job) |
| v3dv_job_destroy(cmd_buffer->state.job); |
| |
| if (cmd_buffer->state.attachments) |
| vk_free(&cmd_buffer->pool->alloc, cmd_buffer->state.attachments); |
| |
| if (cmd_buffer->state.query.end.alloc_count > 0) |
| vk_free(&cmd_buffer->device->alloc, cmd_buffer->state.query.end.states); |
| |
| if (cmd_buffer->push_constants_resource.bo) |
| v3dv_bo_free(cmd_buffer->device, cmd_buffer->push_constants_resource.bo); |
| |
| list_for_each_entry_safe(struct v3dv_cmd_buffer_private_obj, pobj, |
| &cmd_buffer->private_objs, list_link) { |
| cmd_buffer_destroy_private_obj(cmd_buffer, pobj); |
| } |
| |
| if (cmd_buffer->meta.blit.dspool) { |
| v3dv_DestroyDescriptorPool(v3dv_device_to_handle(cmd_buffer->device), |
| cmd_buffer->meta.blit.dspool, |
| &cmd_buffer->device->alloc); |
| } |
| |
| if (cmd_buffer->state.meta.attachments) { |
| assert(cmd_buffer->state.meta.attachment_alloc_count > 0); |
| vk_free(&cmd_buffer->device->alloc, cmd_buffer->state.meta.attachments); |
| } |
| } |
| |
| static void |
| cmd_buffer_destroy(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| list_del(&cmd_buffer->pool_link); |
| cmd_buffer_free_resources(cmd_buffer); |
| vk_free(&cmd_buffer->pool->alloc, cmd_buffer); |
| } |
| |
| void |
| v3dv_job_emit_binning_flush(struct v3dv_job *job) |
| { |
| assert(job); |
| |
| v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(FLUSH)); |
| v3dv_return_if_oom(NULL, job); |
| |
| cl_emit(&job->bcl, FLUSH, flush); |
| } |
| |
| static bool |
| attachment_list_is_subset(struct v3dv_subpass_attachment *l1, uint32_t l1_count, |
| struct v3dv_subpass_attachment *l2, uint32_t l2_count) |
| { |
| for (uint32_t i = 0; i < l1_count; i++) { |
| uint32_t attachment_idx = l1[i].attachment; |
| if (attachment_idx == VK_ATTACHMENT_UNUSED) |
| continue; |
| |
| uint32_t j; |
| for (j = 0; j < l2_count; j++) { |
| if (l2[j].attachment == attachment_idx) |
| break; |
| } |
| if (j == l2_count) |
| return false; |
| } |
| |
| return true; |
| } |
| |
| static bool |
| cmd_buffer_can_merge_subpass(struct v3dv_cmd_buffer *cmd_buffer, |
| uint32_t subpass_idx) |
| { |
| const struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| assert(state->pass); |
| |
| const struct v3dv_physical_device *physical_device = |
| &cmd_buffer->device->instance->physicalDevice; |
| |
| if (cmd_buffer->level != VK_COMMAND_BUFFER_LEVEL_PRIMARY) |
| return false; |
| |
| if (!cmd_buffer->state.job) |
| return false; |
| |
| if (cmd_buffer->state.job->always_flush) |
| return false; |
| |
| if (!physical_device->options.merge_jobs) |
| return false; |
| |
| /* Each render pass starts a new job */ |
| if (subpass_idx == 0) |
| return false; |
| |
| /* Two subpasses can be merged in the same job if we can emit a single RCL |
| * for them (since the RCL includes the END_OF_RENDERING command that |
| * triggers the "render job finished" interrupt). We can do this so long |
| * as both subpasses render against the same attachments. |
| */ |
| assert(state->subpass_idx == subpass_idx - 1); |
| struct v3dv_subpass *prev_subpass = &state->pass->subpasses[state->subpass_idx]; |
| struct v3dv_subpass *subpass = &state->pass->subpasses[subpass_idx]; |
| |
| /* Because the list of subpass attachments can include VK_ATTACHMENT_UNUSED, |
| * we need to check that for each subpass all its used attachments are |
| * used by the other subpass. |
| */ |
| bool compatible = |
| attachment_list_is_subset(prev_subpass->color_attachments, |
| prev_subpass->color_count, |
| subpass->color_attachments, |
| subpass->color_count); |
| if (!compatible) |
| return false; |
| |
| compatible = |
| attachment_list_is_subset(subpass->color_attachments, |
| subpass->color_count, |
| prev_subpass->color_attachments, |
| prev_subpass->color_count); |
| if (!compatible) |
| return false; |
| |
| /* FIXME: resolve attachments */ |
| |
| if (subpass->ds_attachment.attachment != |
| prev_subpass->ds_attachment.attachment) |
| return false; |
| |
| return true; |
| } |
| |
| /** |
| * Computes and sets the job frame tiling information required to setup frame |
| * binning and rendering. |
| */ |
| static struct v3dv_frame_tiling * |
| job_compute_frame_tiling(struct v3dv_job *job, |
| uint32_t width, |
| uint32_t height, |
| uint32_t layers, |
| uint32_t render_target_count, |
| uint8_t max_internal_bpp) |
| { |
| static const uint8_t tile_sizes[] = { |
| 64, 64, |
| 64, 32, |
| 32, 32, |
| 32, 16, |
| 16, 16, |
| }; |
| |
| assert(job); |
| struct v3dv_frame_tiling *tiling = &job->frame_tiling; |
| |
| tiling->width = width; |
| tiling->height = height; |
| tiling->layers = layers; |
| tiling->render_target_count = render_target_count; |
| |
| uint32_t tile_size_index = 0; |
| |
| /* FIXME: MSAA */ |
| |
| if (render_target_count > 2) |
| tile_size_index += 2; |
| else if (render_target_count > 1) |
| tile_size_index += 1; |
| |
| tiling->internal_bpp = max_internal_bpp; |
| tile_size_index += tiling->internal_bpp; |
| assert(tile_size_index < ARRAY_SIZE(tile_sizes)); |
| |
| tiling->tile_width = tile_sizes[tile_size_index * 2]; |
| tiling->tile_height = tile_sizes[tile_size_index * 2 + 1]; |
| |
| tiling->draw_tiles_x = DIV_ROUND_UP(width, tiling->tile_width); |
| tiling->draw_tiles_y = DIV_ROUND_UP(height, tiling->tile_height); |
| |
| /* Size up our supertiles until we get under the limit */ |
| const uint32_t max_supertiles = 256; |
| tiling->supertile_width = 1; |
| tiling->supertile_height = 1; |
| for (;;) { |
| tiling->frame_width_in_supertiles = |
| DIV_ROUND_UP(tiling->draw_tiles_x, tiling->supertile_width); |
| tiling->frame_height_in_supertiles = |
| DIV_ROUND_UP(tiling->draw_tiles_y, tiling->supertile_height); |
| const uint32_t num_supertiles = tiling->frame_width_in_supertiles * |
| tiling->frame_height_in_supertiles; |
| if (num_supertiles < max_supertiles) |
| break; |
| |
| if (tiling->supertile_width < tiling->supertile_height) |
| tiling->supertile_width++; |
| else |
| tiling->supertile_height++; |
| } |
| |
| return tiling; |
| } |
| |
| void |
| v3dv_job_start_frame(struct v3dv_job *job, |
| uint32_t width, |
| uint32_t height, |
| uint32_t layers, |
| uint32_t render_target_count, |
| uint8_t max_internal_bpp) |
| { |
| assert(job); |
| |
| /* Start by computing frame tiling spec for this job */ |
| const struct v3dv_frame_tiling *tiling = |
| job_compute_frame_tiling(job, |
| width, height, layers, |
| render_target_count, max_internal_bpp); |
| |
| v3dv_cl_ensure_space_with_branch(&job->bcl, 256); |
| v3dv_return_if_oom(NULL, job); |
| |
| /* The PTB will request the tile alloc initial size per tile at start |
| * of tile binning. |
| */ |
| uint32_t tile_alloc_size = 64 * tiling->layers * |
| tiling->draw_tiles_x * |
| tiling->draw_tiles_y; |
| |
| /* The PTB allocates in aligned 4k chunks after the initial setup. */ |
| tile_alloc_size = align(tile_alloc_size, 4096); |
| |
| /* Include the first two chunk allocations that the PTB does so that |
| * we definitely clear the OOM condition before triggering one (the HW |
| * won't trigger OOM during the first allocations). |
| */ |
| tile_alloc_size += 8192; |
| |
| /* For performance, allocate some extra initial memory after the PTB's |
| * minimal allocations, so that we hopefully don't have to block the |
| * GPU on the kernel handling an OOM signal. |
| */ |
| tile_alloc_size += 512 * 1024; |
| |
| job->tile_alloc = v3dv_bo_alloc(job->device, tile_alloc_size, |
| "tile_alloc", true); |
| if (!job->tile_alloc) { |
| v3dv_flag_oom(NULL, job); |
| return; |
| } |
| |
| v3dv_job_add_bo(job, job->tile_alloc); |
| |
| const uint32_t tsda_per_tile_size = 256; |
| const uint32_t tile_state_size = tiling->layers * |
| tiling->draw_tiles_x * |
| tiling->draw_tiles_y * |
| tsda_per_tile_size; |
| job->tile_state = v3dv_bo_alloc(job->device, tile_state_size, "TSDA", true); |
| if (!job->tile_state) { |
| v3dv_flag_oom(NULL, job); |
| return; |
| } |
| |
| v3dv_job_add_bo(job, job->tile_state); |
| |
| /* This must go before the binning mode configuration. It is |
| * required for layered framebuffers to work. |
| */ |
| cl_emit(&job->bcl, NUMBER_OF_LAYERS, config) { |
| config.number_of_layers = layers; |
| } |
| |
| cl_emit(&job->bcl, TILE_BINNING_MODE_CFG, config) { |
| config.width_in_pixels = tiling->width; |
| config.height_in_pixels = tiling->height; |
| config.number_of_render_targets = MAX2(tiling->render_target_count, 1); |
| config.multisample_mode_4x = false; /* FIXME */ |
| config.maximum_bpp_of_all_render_targets = tiling->internal_bpp; |
| } |
| |
| /* There's definitely nothing in the VCD cache we want. */ |
| cl_emit(&job->bcl, FLUSH_VCD_CACHE, bin); |
| |
| /* "Binning mode lists must have a Start Tile Binning item (6) after |
| * any prefix state data before the binning list proper starts." |
| */ |
| cl_emit(&job->bcl, START_TILE_BINNING, bin); |
| |
| job->ez_state = VC5_EZ_UNDECIDED; |
| job->first_ez_state = VC5_EZ_UNDECIDED; |
| } |
| |
| static void |
| cmd_buffer_end_render_pass_frame(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| assert(cmd_buffer->state.job); |
| |
| /* Typically, we have a single job for each subpass and we emit the job's RCL |
| * here when we are ending the frame for the subpass. However, some commands |
| * such as vkCmdClearAttachments need to run in their own separate job and |
| * they emit their own RCL even if they execute inside a subpass. In this |
| * scenario, we don't want to emit subpass RCL when we end the frame for |
| * those jobs, so we only emit the subpass RCL if the job has not recorded |
| * any RCL commands of its own. |
| */ |
| if (v3dv_cl_offset(&cmd_buffer->state.job->rcl) == 0) |
| cmd_buffer_emit_render_pass_rcl(cmd_buffer); |
| |
| v3dv_job_emit_binning_flush(cmd_buffer->state.job); |
| } |
| |
| static void |
| cmd_buffer_end_render_pass_secondary(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| assert(cmd_buffer->state.job); |
| v3dv_cl_ensure_space_with_branch(&cmd_buffer->state.job->bcl, |
| cl_packet_length(RETURN_FROM_SUB_LIST)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| cl_emit(&cmd_buffer->state.job->bcl, RETURN_FROM_SUB_LIST, ret); |
| } |
| |
| struct v3dv_job * |
| v3dv_cmd_buffer_create_cpu_job(struct v3dv_device *device, |
| enum v3dv_job_type type, |
| struct v3dv_cmd_buffer *cmd_buffer, |
| uint32_t subpass_idx) |
| { |
| struct v3dv_job *job = vk_zalloc(&device->alloc, |
| sizeof(struct v3dv_job), 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); |
| if (!job) { |
| v3dv_flag_oom(cmd_buffer, NULL); |
| return NULL; |
| } |
| |
| v3dv_job_init(job, type, device, cmd_buffer, subpass_idx); |
| return job; |
| } |
| |
| static void |
| cmd_buffer_add_cpu_jobs_for_pending_state(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| |
| if (state->query.end.used_count > 0) { |
| const uint32_t query_count = state->query.end.used_count; |
| for (uint32_t i = 0; i < query_count; i++) { |
| assert(i < state->query.end.used_count); |
| struct v3dv_job *job = |
| v3dv_cmd_buffer_create_cpu_job(cmd_buffer->device, |
| V3DV_JOB_TYPE_CPU_END_QUERY, |
| cmd_buffer, -1); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| job->cpu.query_end = state->query.end.states[i]; |
| list_addtail(&job->list_link, &cmd_buffer->jobs); |
| } |
| } |
| } |
| |
| void |
| v3dv_cmd_buffer_finish_job(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_job *job = cmd_buffer->state.job; |
| if (!job) |
| return; |
| |
| if (cmd_buffer->state.oom) { |
| v3dv_job_destroy(job); |
| cmd_buffer->state.job = NULL; |
| return; |
| } |
| |
| /* If we have created a job for a command buffer then we should have |
| * recorded something into it: if the job was started in a render pass, it |
| * should at least have the start frame commands, otherwise, it should have |
| * a transfer command. The only exception are secondary command buffers |
| * inside a render pass. |
| */ |
| assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY || |
| v3dv_cl_offset(&job->bcl) > 0); |
| |
| /* When we merge multiple subpasses into the same job we must only emit one |
| * RCL, so we do that here, when we decided that we need to finish the job. |
| * Any rendering that happens outside a render pass is never merged, so |
| * the RCL should have been emitted by the time we got here. |
| */ |
| assert(v3dv_cl_offset(&job->rcl) != 0 || cmd_buffer->state.pass); |
| |
| /* If we are finishing a job inside a render pass we have two scenarios: |
| * |
| * 1. It is a regular CL, in which case we will submit the job to the GPU, |
| * so we may need to generate an RCL and add a binning flush. |
| * |
| * 2. It is a partial CL recorded in a secondary command buffer, in which |
| * case we are not submitting it directly to the GPU but rather branch to |
| * it from a primary command buffer. In this case we just want to end |
| * the BCL with a RETURN_FROM_SUB_LIST and the RCL and binning flush |
| * will be the primary job that branches to this CL. |
| */ |
| if (cmd_buffer->state.pass) { |
| if (job->type == V3DV_JOB_TYPE_GPU_CL) { |
| cmd_buffer_end_render_pass_frame(cmd_buffer); |
| } else { |
| assert(job->type == V3DV_JOB_TYPE_GPU_CL_SECONDARY); |
| cmd_buffer_end_render_pass_secondary(cmd_buffer); |
| } |
| } |
| |
| list_addtail(&job->list_link, &cmd_buffer->jobs); |
| cmd_buffer->state.job = NULL; |
| |
| /* If we have recorded any state with this last GPU job that requires to |
| * emit CPU jobs after the job is completed, add them now. The only |
| * exception is secondary command buffers inside a render pass, because in |
| * that case we want to defer this until we finish recording the primary |
| * job into which we execute the secondary. |
| */ |
| if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY || |
| !cmd_buffer->state.pass) { |
| cmd_buffer_add_cpu_jobs_for_pending_state(cmd_buffer); |
| } |
| } |
| |
| void |
| v3dv_job_init(struct v3dv_job *job, |
| enum v3dv_job_type type, |
| struct v3dv_device *device, |
| struct v3dv_cmd_buffer *cmd_buffer, |
| int32_t subpass_idx) |
| { |
| assert(job); |
| |
| job->type = type; |
| |
| job->device = device; |
| job->cmd_buffer = cmd_buffer; |
| |
| list_inithead(&job->list_link); |
| |
| if (type == V3DV_JOB_TYPE_GPU_CL || |
| type == V3DV_JOB_TYPE_GPU_CL_SECONDARY || |
| type == V3DV_JOB_TYPE_GPU_CSD) { |
| job->bos = |
| _mesa_set_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal); |
| job->bo_count = 0; |
| |
| v3dv_cl_init(job, &job->indirect); |
| |
| if (V3D_DEBUG & V3D_DEBUG_ALWAYS_FLUSH) |
| job->always_flush = true; |
| } |
| |
| if (type == V3DV_JOB_TYPE_GPU_CL || |
| type == V3DV_JOB_TYPE_GPU_CL_SECONDARY) { |
| v3dv_cl_init(job, &job->bcl); |
| v3dv_cl_init(job, &job->rcl); |
| } |
| |
| if (cmd_buffer) { |
| /* Flag all state as dirty. Generally, we need to re-emit state for each |
| * new job. |
| * |
| * FIXME: there may be some exceptions, in which case we could skip some |
| * bits. |
| */ |
| cmd_buffer->state.dirty = ~0; |
| |
| /* Keep track of the first subpass that we are recording in this new job. |
| * We will use this when we emit the RCL to decide how to emit our loads |
| * and stores. |
| */ |
| if (cmd_buffer->state.pass) |
| job->first_subpass = subpass_idx; |
| } |
| } |
| |
| struct v3dv_job * |
| v3dv_cmd_buffer_start_job(struct v3dv_cmd_buffer *cmd_buffer, |
| int32_t subpass_idx, |
| enum v3dv_job_type type) |
| { |
| /* Don't create a new job if we can merge the current subpass into |
| * the current job. |
| */ |
| if (cmd_buffer->state.pass && |
| subpass_idx != -1 && |
| cmd_buffer_can_merge_subpass(cmd_buffer, subpass_idx)) { |
| cmd_buffer->state.job->is_subpass_finish = false; |
| return cmd_buffer->state.job; |
| } |
| |
| /* Ensure we are not starting a new job without finishing a previous one */ |
| if (cmd_buffer->state.job != NULL) |
| v3dv_cmd_buffer_finish_job(cmd_buffer); |
| |
| assert(cmd_buffer->state.job == NULL); |
| struct v3dv_job *job = vk_zalloc(&cmd_buffer->device->alloc, |
| sizeof(struct v3dv_job), 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); |
| |
| cmd_buffer->state.job = job; |
| |
| if (!job) { |
| fprintf(stderr, "Error: failed to allocate CPU memory for job\n"); |
| v3dv_flag_oom(cmd_buffer, NULL); |
| return NULL; |
| } |
| |
| v3dv_job_init(job, type, cmd_buffer->device, cmd_buffer, subpass_idx); |
| |
| return job; |
| } |
| |
| static VkResult |
| cmd_buffer_reset(struct v3dv_cmd_buffer *cmd_buffer, |
| VkCommandBufferResetFlags flags) |
| { |
| if (cmd_buffer->status != V3DV_CMD_BUFFER_STATUS_INITIALIZED) { |
| struct v3dv_device *device = cmd_buffer->device; |
| struct v3dv_cmd_pool *pool = cmd_buffer->pool; |
| VkCommandBufferLevel level = cmd_buffer->level; |
| |
| /* cmd_buffer_init below will re-add the command buffer to the pool |
| * so remove it here so we don't end up adding it again. |
| */ |
| list_del(&cmd_buffer->pool_link); |
| |
| /* FIXME: For now we always free all resources as if |
| * VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT was set. |
| */ |
| if (cmd_buffer->status != V3DV_CMD_BUFFER_STATUS_NEW) |
| cmd_buffer_free_resources(cmd_buffer); |
| |
| cmd_buffer_init(cmd_buffer, device, pool, level); |
| } |
| |
| assert(cmd_buffer->status == V3DV_CMD_BUFFER_STATUS_INITIALIZED); |
| return VK_SUCCESS; |
| } |
| |
| VkResult |
| v3dv_AllocateCommandBuffers(VkDevice _device, |
| const VkCommandBufferAllocateInfo *pAllocateInfo, |
| VkCommandBuffer *pCommandBuffers) |
| { |
| V3DV_FROM_HANDLE(v3dv_device, device, _device); |
| V3DV_FROM_HANDLE(v3dv_cmd_pool, pool, pAllocateInfo->commandPool); |
| |
| VkResult result = VK_SUCCESS; |
| uint32_t i; |
| |
| for (i = 0; i < pAllocateInfo->commandBufferCount; i++) { |
| result = cmd_buffer_create(device, pool, pAllocateInfo->level, |
| &pCommandBuffers[i]); |
| if (result != VK_SUCCESS) |
| break; |
| } |
| |
| if (result != VK_SUCCESS) { |
| v3dv_FreeCommandBuffers(_device, pAllocateInfo->commandPool, |
| i, pCommandBuffers); |
| for (i = 0; i < pAllocateInfo->commandBufferCount; i++) |
| pCommandBuffers[i] = VK_NULL_HANDLE; |
| } |
| |
| return result; |
| } |
| |
| void |
| v3dv_FreeCommandBuffers(VkDevice device, |
| VkCommandPool commandPool, |
| uint32_t commandBufferCount, |
| const VkCommandBuffer *pCommandBuffers) |
| { |
| for (uint32_t i = 0; i < commandBufferCount; i++) { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, pCommandBuffers[i]); |
| |
| if (!cmd_buffer) |
| continue; |
| |
| cmd_buffer_destroy(cmd_buffer); |
| } |
| } |
| |
| void |
| v3dv_DestroyCommandPool(VkDevice _device, |
| VkCommandPool commandPool, |
| const VkAllocationCallbacks *pAllocator) |
| { |
| V3DV_FROM_HANDLE(v3dv_device, device, _device); |
| V3DV_FROM_HANDLE(v3dv_cmd_pool, pool, commandPool); |
| |
| if (!pool) |
| return; |
| |
| list_for_each_entry_safe(struct v3dv_cmd_buffer, cmd_buffer, |
| &pool->cmd_buffers, pool_link) { |
| cmd_buffer_destroy(cmd_buffer); |
| } |
| |
| vk_free2(&device->alloc, pAllocator, pool); |
| } |
| |
| static VkResult |
| cmd_buffer_begin_render_pass_secondary( |
| struct v3dv_cmd_buffer *cmd_buffer, |
| const VkCommandBufferInheritanceInfo *inheritance_info) |
| { |
| assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY); |
| assert(cmd_buffer->usage_flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT); |
| assert(inheritance_info); |
| |
| cmd_buffer->state.pass = |
| v3dv_render_pass_from_handle(inheritance_info->renderPass); |
| assert(cmd_buffer->state.pass); |
| |
| cmd_buffer->state.framebuffer = |
| v3dv_framebuffer_from_handle(inheritance_info->framebuffer); |
| |
| /* Secondaries that execute inside a render pass won't start subpasses |
| * so we want to create a job for them here. |
| */ |
| assert(inheritance_info->subpass < cmd_buffer->state.pass->subpass_count); |
| struct v3dv_job *job = |
| v3dv_cmd_buffer_start_job(cmd_buffer, inheritance_info->subpass, |
| V3DV_JOB_TYPE_GPU_CL_SECONDARY); |
| if (!job) { |
| v3dv_flag_oom(cmd_buffer, NULL); |
| return VK_ERROR_OUT_OF_HOST_MEMORY; |
| } |
| |
| cmd_buffer->state.subpass_idx = inheritance_info->subpass; |
| |
| /* Secondary command buffers don't know about the render area, but our |
| * scissor setup accounts for it, so let's make sure we make it large |
| * enough that it doesn't actually constrain any rendering. This should |
| * be fine, since the Vulkan spec states: |
| * |
| * "The application must ensure (using scissor if necessary) that all |
| * rendering is contained within the render area." |
| * |
| * FIXME: setup constants for the max framebuffer dimensions and use them |
| * here and when filling in VkPhysicalDeviceLimits. |
| */ |
| const struct v3dv_framebuffer *framebuffer = cmd_buffer->state.framebuffer; |
| cmd_buffer->state.render_area.offset.x = 0; |
| cmd_buffer->state.render_area.offset.y = 0; |
| cmd_buffer->state.render_area.extent.width = |
| framebuffer ? framebuffer->width : 4096; |
| cmd_buffer->state.render_area.extent.height = |
| framebuffer ? framebuffer->height : 4096; |
| |
| return VK_SUCCESS; |
| } |
| |
| VkResult |
| v3dv_BeginCommandBuffer(VkCommandBuffer commandBuffer, |
| const VkCommandBufferBeginInfo *pBeginInfo) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| |
| /* If this is the first vkBeginCommandBuffer, we must initialize the |
| * command buffer's state. Otherwise, we must reset its state. In both |
| * cases we reset it. |
| */ |
| VkResult result = cmd_buffer_reset(cmd_buffer, 0); |
| if (result != VK_SUCCESS) |
| return result; |
| |
| assert(cmd_buffer->status == V3DV_CMD_BUFFER_STATUS_INITIALIZED); |
| |
| cmd_buffer->usage_flags = pBeginInfo->flags; |
| |
| if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY) { |
| if (pBeginInfo->flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT) { |
| result = |
| cmd_buffer_begin_render_pass_secondary(cmd_buffer, |
| pBeginInfo->pInheritanceInfo); |
| if (result != VK_SUCCESS) |
| return result; |
| } |
| |
| /* If the primary may have an active occlusion query we need to honor |
| * that in the secondary. |
| */ |
| if (pBeginInfo->pInheritanceInfo->occlusionQueryEnable) |
| cmd_buffer->state.dirty &= ~V3DV_CMD_DIRTY_OCCLUSION_QUERY; |
| } |
| |
| cmd_buffer->status = V3DV_CMD_BUFFER_STATUS_RECORDING; |
| |
| return VK_SUCCESS; |
| } |
| |
| VkResult |
| v3dv_ResetCommandBuffer(VkCommandBuffer commandBuffer, |
| VkCommandBufferResetFlags flags) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| return cmd_buffer_reset(cmd_buffer, flags); |
| } |
| |
| VkResult |
| v3dv_ResetCommandPool(VkDevice device, |
| VkCommandPool commandPool, |
| VkCommandPoolResetFlags flags) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_pool, pool, commandPool); |
| |
| VkCommandBufferResetFlags reset_flags = 0; |
| if (flags & VK_COMMAND_POOL_RESET_RELEASE_RESOURCES_BIT) |
| reset_flags = VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT; |
| list_for_each_entry_safe(struct v3dv_cmd_buffer, cmd_buffer, |
| &pool->cmd_buffers, pool_link) { |
| cmd_buffer_reset(cmd_buffer, reset_flags); |
| } |
| |
| return VK_SUCCESS; |
| } |
| static void |
| emit_clip_window(struct v3dv_job *job, const VkRect2D *rect) |
| { |
| assert(job); |
| |
| v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(CLIP_WINDOW)); |
| v3dv_return_if_oom(NULL, job); |
| |
| cl_emit(&job->bcl, CLIP_WINDOW, clip) { |
| clip.clip_window_left_pixel_coordinate = rect->offset.x; |
| clip.clip_window_bottom_pixel_coordinate = rect->offset.y; |
| clip.clip_window_width_in_pixels = rect->extent.width; |
| clip.clip_window_height_in_pixels = rect->extent.height; |
| } |
| } |
| |
| /* Checks whether the render area rectangle covers a region that is aligned to |
| * tile boundaries, which means that for all tiles covered by the render area |
| * region, there are no uncovered pixels (unless they are also outside the |
| * framebuffer). |
| */ |
| static void |
| cmd_buffer_update_tile_alignment(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| /* Render areas and scissor/viewport are only relevant inside render passes, |
| * otherwise we are dealing with transfer operations where these elements |
| * don't apply. |
| */ |
| assert(cmd_buffer->state.pass); |
| const VkRect2D *rect = &cmd_buffer->state.render_area; |
| |
| /* We should only call this at the beginning of a subpass so we should |
| * always have framebuffer information available. |
| */ |
| assert(cmd_buffer->state.framebuffer); |
| |
| const VkExtent2D fb_extent = { |
| .width = cmd_buffer->state.framebuffer->width, |
| .height = cmd_buffer->state.framebuffer->height |
| }; |
| |
| VkExtent2D granularity; |
| v3dv_subpass_get_granularity(cmd_buffer->state.pass, |
| cmd_buffer->state.subpass_idx, |
| &granularity); |
| |
| cmd_buffer->state.tile_aligned_render_area = |
| rect->offset.x % granularity.width == 0 && |
| rect->offset.y % granularity.height == 0 && |
| (rect->extent.width % granularity.width == 0 || |
| rect->offset.x + rect->extent.width >= fb_extent.width) && |
| (rect->extent.height % granularity.height == 0 || |
| rect->offset.y + rect->extent.height >= fb_extent.height); |
| |
| if (!cmd_buffer->state.tile_aligned_render_area) { |
| perf_debug("Render area for subpass %d of render pass %p doesn't " |
| "match render pass granularity.\n", |
| cmd_buffer->state.subpass_idx, cmd_buffer->state.pass); |
| } |
| } |
| |
| void |
| v3dv_get_hw_clear_color(const VkClearColorValue *color, |
| uint32_t internal_type, |
| uint32_t internal_size, |
| uint32_t *hw_color) |
| { |
| union util_color uc; |
| switch (internal_type) { |
| case V3D_INTERNAL_TYPE_8: |
| util_pack_color(color->float32, PIPE_FORMAT_R8G8B8A8_UNORM, &uc); |
| memcpy(hw_color, uc.ui, internal_size); |
| break; |
| case V3D_INTERNAL_TYPE_8I: |
| case V3D_INTERNAL_TYPE_8UI: |
| hw_color[0] = ((color->uint32[0] & 0xff) | |
| (color->uint32[1] & 0xff) << 8 | |
| (color->uint32[2] & 0xff) << 16 | |
| (color->uint32[3] & 0xff) << 24); |
| break; |
| case V3D_INTERNAL_TYPE_16F: |
| util_pack_color(color->float32, PIPE_FORMAT_R16G16B16A16_FLOAT, &uc); |
| memcpy(hw_color, uc.ui, internal_size); |
| break; |
| case V3D_INTERNAL_TYPE_16I: |
| case V3D_INTERNAL_TYPE_16UI: |
| hw_color[0] = ((color->uint32[0] & 0xffff) | color->uint32[1] << 16); |
| hw_color[1] = ((color->uint32[2] & 0xffff) | color->uint32[3] << 16); |
| break; |
| case V3D_INTERNAL_TYPE_32F: |
| case V3D_INTERNAL_TYPE_32I: |
| case V3D_INTERNAL_TYPE_32UI: |
| memcpy(hw_color, color->uint32, internal_size); |
| break; |
| } |
| } |
| |
| static void |
| cmd_buffer_state_set_attachment_clear_color(struct v3dv_cmd_buffer *cmd_buffer, |
| uint32_t attachment_idx, |
| const VkClearColorValue *color) |
| { |
| assert(attachment_idx < cmd_buffer->state.pass->attachment_count); |
| |
| const struct v3dv_render_pass_attachment *attachment = |
| &cmd_buffer->state.pass->attachments[attachment_idx]; |
| |
| uint32_t internal_type, internal_bpp; |
| const struct v3dv_format *format = v3dv_get_format(attachment->desc.format); |
| v3dv_get_internal_type_bpp_for_output_format(format->rt_type, |
| &internal_type, |
| &internal_bpp); |
| |
| uint32_t internal_size = 4 << internal_bpp; |
| |
| struct v3dv_cmd_buffer_attachment_state *attachment_state = |
| &cmd_buffer->state.attachments[attachment_idx]; |
| |
| v3dv_get_hw_clear_color(color, internal_type, internal_size, |
| &attachment_state->clear_value.color[0]); |
| |
| attachment_state->vk_clear_value.color = *color; |
| } |
| |
| static void |
| cmd_buffer_state_set_attachment_clear_depth_stencil( |
| struct v3dv_cmd_buffer *cmd_buffer, |
| uint32_t attachment_idx, |
| bool clear_depth, bool clear_stencil, |
| const VkClearDepthStencilValue *ds) |
| { |
| struct v3dv_cmd_buffer_attachment_state *attachment_state = |
| &cmd_buffer->state.attachments[attachment_idx]; |
| |
| if (clear_depth) |
| attachment_state->clear_value.z = ds->depth; |
| |
| if (clear_stencil) |
| attachment_state->clear_value.s = ds->stencil; |
| |
| attachment_state->vk_clear_value.depthStencil = *ds; |
| } |
| |
| static void |
| cmd_buffer_state_set_clear_values(struct v3dv_cmd_buffer *cmd_buffer, |
| uint32_t count, const VkClearValue *values) |
| { |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| const struct v3dv_render_pass *pass = state->pass; |
| |
| /* There could be less clear values than attachments in the render pass, in |
| * which case we only want to process as many as we have, or there could be |
| * more, in which case we want to ignore those for which we don't have a |
| * corresponding attachment. |
| */ |
| count = MIN2(count, pass->attachment_count); |
| for (uint32_t i = 0; i < count; i++) { |
| const struct v3dv_render_pass_attachment *attachment = |
| &pass->attachments[i]; |
| |
| if (attachment->desc.loadOp != VK_ATTACHMENT_LOAD_OP_CLEAR) |
| continue; |
| |
| VkImageAspectFlags aspects = vk_format_aspects(attachment->desc.format); |
| if (aspects & VK_IMAGE_ASPECT_COLOR_BIT) { |
| cmd_buffer_state_set_attachment_clear_color(cmd_buffer, i, |
| &values[i].color); |
| } else if (aspects & (VK_IMAGE_ASPECT_DEPTH_BIT | |
| VK_IMAGE_ASPECT_STENCIL_BIT)) { |
| cmd_buffer_state_set_attachment_clear_depth_stencil( |
| cmd_buffer, i, |
| aspects & VK_IMAGE_ASPECT_DEPTH_BIT, |
| aspects & VK_IMAGE_ASPECT_STENCIL_BIT, |
| &values[i].depthStencil); |
| } |
| } |
| } |
| |
| static void |
| cmd_buffer_init_render_pass_attachment_state(struct v3dv_cmd_buffer *cmd_buffer, |
| const VkRenderPassBeginInfo *pRenderPassBegin) |
| { |
| cmd_buffer_state_set_clear_values(cmd_buffer, |
| pRenderPassBegin->clearValueCount, |
| pRenderPassBegin->pClearValues); |
| } |
| |
| static void |
| cmd_buffer_ensure_render_pass_attachment_state(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| const struct v3dv_render_pass *pass = state->pass; |
| |
| if (state->attachment_alloc_count < pass->attachment_count) { |
| if (state->attachments > 0) { |
| assert(state->attachment_alloc_count > 0); |
| vk_free(&cmd_buffer->device->alloc, state->attachments); |
| } |
| |
| uint32_t size = sizeof(struct v3dv_cmd_buffer_attachment_state) * |
| pass->attachment_count; |
| state->attachments = vk_zalloc(&cmd_buffer->device->alloc, size, 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); |
| if (!state->attachments) { |
| v3dv_flag_oom(cmd_buffer, NULL); |
| return; |
| } |
| state->attachment_alloc_count = pass->attachment_count; |
| } |
| |
| assert(state->attachment_alloc_count >= pass->attachment_count); |
| } |
| |
| void |
| v3dv_CmdBeginRenderPass(VkCommandBuffer commandBuffer, |
| const VkRenderPassBeginInfo *pRenderPassBegin, |
| VkSubpassContents contents) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| V3DV_FROM_HANDLE(v3dv_render_pass, pass, pRenderPassBegin->renderPass); |
| V3DV_FROM_HANDLE(v3dv_framebuffer, framebuffer, pRenderPassBegin->framebuffer); |
| |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| state->pass = pass; |
| state->framebuffer = framebuffer; |
| |
| cmd_buffer_ensure_render_pass_attachment_state(cmd_buffer); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cmd_buffer_init_render_pass_attachment_state(cmd_buffer, pRenderPassBegin); |
| |
| state->render_area = pRenderPassBegin->renderArea; |
| |
| /* If our render area is smaller than the current clip window we will have |
| * to emit a new clip window to constraint it to the render area. |
| */ |
| uint32_t min_render_x = state->render_area.offset.x; |
| uint32_t min_render_y = state->render_area.offset.x; |
| uint32_t max_render_x = min_render_x + state->render_area.extent.width - 1; |
| uint32_t max_render_y = min_render_y + state->render_area.extent.height - 1; |
| uint32_t min_clip_x = state->clip_window.offset.x; |
| uint32_t min_clip_y = state->clip_window.offset.y; |
| uint32_t max_clip_x = min_clip_x + state->clip_window.extent.width - 1; |
| uint32_t max_clip_y = min_clip_y + state->clip_window.extent.height - 1; |
| if (min_render_x > min_clip_x || min_render_y > min_clip_y || |
| max_render_x < max_clip_x || max_render_y < max_clip_y) { |
| state->dirty |= V3DV_CMD_DIRTY_SCISSOR; |
| } |
| |
| /* Setup for first subpass */ |
| v3dv_cmd_buffer_subpass_start(cmd_buffer, 0); |
| } |
| |
| void |
| v3dv_CmdNextSubpass(VkCommandBuffer commandBuffer, VkSubpassContents contents) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| assert(state->subpass_idx < state->pass->subpass_count - 1); |
| |
| /* Finish the previous subpass */ |
| v3dv_cmd_buffer_subpass_finish(cmd_buffer); |
| |
| /* Start the next subpass */ |
| v3dv_cmd_buffer_subpass_start(cmd_buffer, state->subpass_idx + 1); |
| } |
| |
| void |
| v3dv_render_pass_setup_render_target(struct v3dv_cmd_buffer *cmd_buffer, |
| int rt, |
| uint32_t *rt_bpp, |
| uint32_t *rt_type, |
| uint32_t *rt_clamp) |
| { |
| const struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| |
| assert(state->subpass_idx < state->pass->subpass_count); |
| const struct v3dv_subpass *subpass = |
| &state->pass->subpasses[state->subpass_idx]; |
| |
| if (rt >= subpass->color_count) |
| return; |
| |
| struct v3dv_subpass_attachment *attachment = &subpass->color_attachments[rt]; |
| const uint32_t attachment_idx = attachment->attachment; |
| if (attachment_idx == VK_ATTACHMENT_UNUSED) |
| return; |
| |
| const struct v3dv_framebuffer *framebuffer = state->framebuffer; |
| assert(attachment_idx < framebuffer->attachment_count); |
| struct v3dv_image_view *iview = framebuffer->attachments[attachment_idx]; |
| assert(iview->aspects & VK_IMAGE_ASPECT_COLOR_BIT); |
| |
| *rt_bpp = iview->internal_bpp; |
| *rt_type = iview->internal_type; |
| *rt_clamp =vk_format_is_int(iview->vk_format) ? |
| V3D_RENDER_TARGET_CLAMP_INT : V3D_RENDER_TARGET_CLAMP_NONE; |
| } |
| |
| static void |
| cmd_buffer_render_pass_emit_load(struct v3dv_cmd_buffer *cmd_buffer, |
| struct v3dv_cl *cl, |
| struct v3dv_image_view *iview, |
| uint32_t layer, |
| uint32_t buffer) |
| { |
| const struct v3dv_image *image = iview->image; |
| const struct v3d_resource_slice *slice = &image->slices[iview->base_level]; |
| uint32_t layer_offset = v3dv_layer_offset(image, |
| iview->base_level, |
| iview->first_layer + layer); |
| |
| cl_emit(cl, LOAD_TILE_BUFFER_GENERAL, load) { |
| load.buffer_to_load = buffer; |
| load.address = v3dv_cl_address(image->mem->bo, layer_offset); |
| |
| load.input_image_format = iview->format->rt_type; |
| load.r_b_swap = iview->swap_rb; |
| load.memory_format = slice->tiling; |
| |
| if (slice->tiling == VC5_TILING_UIF_NO_XOR || |
| slice->tiling == VC5_TILING_UIF_XOR) { |
| load.height_in_ub_or_stride = |
| slice->padded_height_of_output_image_in_uif_blocks; |
| } else if (slice->tiling == VC5_TILING_RASTER) { |
| load.height_in_ub_or_stride = slice->stride; |
| } |
| |
| if (image->samples > VK_SAMPLE_COUNT_1_BIT) |
| load.decimate_mode = V3D_DECIMATE_MODE_ALL_SAMPLES; |
| else |
| load.decimate_mode = V3D_DECIMATE_MODE_SAMPLE_0; |
| } |
| } |
| |
| static void |
| cmd_buffer_render_pass_emit_loads(struct v3dv_cmd_buffer *cmd_buffer, |
| struct v3dv_cl *cl, |
| uint32_t layer) |
| { |
| const struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| const struct v3dv_framebuffer *framebuffer = state->framebuffer; |
| const struct v3dv_render_pass *pass = state->pass; |
| const struct v3dv_subpass *subpass = &pass->subpasses[state->subpass_idx]; |
| |
| for (uint32_t i = 0; i < subpass->color_count; i++) { |
| uint32_t attachment_idx = subpass->color_attachments[i].attachment; |
| |
| if (attachment_idx == VK_ATTACHMENT_UNUSED) |
| continue; |
| |
| const struct v3dv_render_pass_attachment *attachment = |
| &state->pass->attachments[attachment_idx]; |
| |
| /* According to the Vulkan spec: |
| * |
| * "The load operation for each sample in an attachment happens before |
| * any recorded command which accesses the sample in the first subpass |
| * where the attachment is used." |
| * |
| * If the load operation is CLEAR, we must only clear once on the first |
| * subpass that uses the attachment (and in that case we don't LOAD). |
| * After that, we always want to load so we don't lose any rendering done |
| * by a previous subpass to the same attachment. We also want to load |
| * if the current job is continuing subpass work started by a previous |
| * job, for the same reason. |
| * |
| * If the render area is not aligned to tile boundaries then we have |
| * tiles which are partially covered by it. In this case, we need to |
| * load the tiles so we can preserve the pixels that are outside the |
| * render area for any such tiles. |
| */ |
| assert(state->job->first_subpass >= attachment->first_subpass); |
| bool needs_load = |
| state->job->first_subpass > attachment->first_subpass || |
| state->job->is_subpass_continue || |
| attachment->desc.loadOp == VK_ATTACHMENT_LOAD_OP_LOAD || |
| !state->tile_aligned_render_area; |
| |
| if (needs_load) { |
| struct v3dv_image_view *iview = framebuffer->attachments[attachment_idx]; |
| cmd_buffer_render_pass_emit_load(cmd_buffer, cl, iview, |
| layer, RENDER_TARGET_0 + i); |
| } |
| } |
| |
| uint32_t ds_attachment_idx = subpass->ds_attachment.attachment; |
| if (ds_attachment_idx != VK_ATTACHMENT_UNUSED) { |
| const struct v3dv_render_pass_attachment *ds_attachment = |
| &state->pass->attachments[ds_attachment_idx]; |
| |
| assert(state->job->first_subpass >= ds_attachment->first_subpass); |
| bool needs_load = |
| state->job->first_subpass > ds_attachment->first_subpass || |
| state->job->is_subpass_continue || |
| ds_attachment->desc.loadOp == VK_ATTACHMENT_LOAD_OP_LOAD || |
| ds_attachment->desc.stencilLoadOp == VK_ATTACHMENT_LOAD_OP_LOAD || |
| !state->tile_aligned_render_area; |
| |
| if (needs_load) { |
| struct v3dv_image_view *iview = |
| framebuffer->attachments[ds_attachment_idx]; |
| /* From the Vulkan spec: |
| * |
| * "When an image view of a depth/stencil image is used as a |
| * depth/stencil framebuffer attachment, the aspectMask is ignored |
| * and both depth and stencil image subresources are used." |
| */ |
| const uint32_t zs_buffer = |
| v3dv_zs_buffer_from_vk_format(iview->image->vk_format); |
| cmd_buffer_render_pass_emit_load(cmd_buffer, cl, |
| iview, layer, zs_buffer); |
| } |
| } |
| |
| cl_emit(cl, END_OF_LOADS, end); |
| } |
| |
| static void |
| cmd_buffer_render_pass_emit_store(struct v3dv_cmd_buffer *cmd_buffer, |
| struct v3dv_cl *cl, |
| uint32_t attachment_idx, |
| uint32_t layer, |
| uint32_t buffer, |
| bool clear) |
| { |
| const struct v3dv_image_view *iview = |
| cmd_buffer->state.framebuffer->attachments[attachment_idx]; |
| const struct v3dv_image *image = iview->image; |
| const struct v3d_resource_slice *slice = &image->slices[iview->base_level]; |
| uint32_t layer_offset = v3dv_layer_offset(image, |
| iview->base_level, |
| iview->first_layer + layer); |
| |
| cl_emit(cl, STORE_TILE_BUFFER_GENERAL, store) { |
| store.buffer_to_store = buffer; |
| store.address = v3dv_cl_address(image->mem->bo, layer_offset); |
| store.clear_buffer_being_stored = clear; |
| |
| store.output_image_format = iview->format->rt_type; |
| store.r_b_swap = iview->swap_rb; |
| store.memory_format = slice->tiling; |
| |
| if (slice->tiling == VC5_TILING_UIF_NO_XOR || |
| slice->tiling == VC5_TILING_UIF_XOR) { |
| store.height_in_ub_or_stride = |
| slice->padded_height_of_output_image_in_uif_blocks; |
| } else if (slice->tiling == VC5_TILING_RASTER) { |
| store.height_in_ub_or_stride = slice->stride; |
| } |
| |
| if (image->samples > VK_SAMPLE_COUNT_1_BIT) |
| store.decimate_mode = V3D_DECIMATE_MODE_ALL_SAMPLES; |
| else |
| store.decimate_mode = V3D_DECIMATE_MODE_SAMPLE_0; |
| } |
| } |
| |
| static void |
| cmd_buffer_render_pass_emit_stores(struct v3dv_cmd_buffer *cmd_buffer, |
| struct v3dv_cl *cl, |
| uint32_t layer) |
| { |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| const struct v3dv_subpass *subpass = |
| &state->pass->subpasses[state->subpass_idx]; |
| |
| bool has_stores = false; |
| bool use_per_buffer_clear = true; |
| |
| /* FIXME: separate stencil */ |
| uint32_t ds_attachment_idx = subpass->ds_attachment.attachment; |
| if (ds_attachment_idx != VK_ATTACHMENT_UNUSED) { |
| const struct v3dv_render_pass_attachment *ds_attachment = |
| &state->pass->attachments[ds_attachment_idx]; |
| |
| assert(state->job->first_subpass >= ds_attachment->first_subpass); |
| assert(state->subpass_idx >= ds_attachment->first_subpass); |
| assert(state->subpass_idx <= ds_attachment->last_subpass); |
| |
| /* From the Vulkan spec, VkImageSubresourceRange: |
| * |
| * "When an image view of a depth/stencil image is used as a |
| * depth/stencil framebuffer attachment, the aspectMask is ignored |
| * and both depth and stencil image subresources are used." |
| * |
| * So we ignore the aspects from the subresource range of the image view |
| * for the depth/stencil attachment, but we still need to restrict this |
| * to aspects that actually exist in the image. |
| */ |
| const VkImageAspectFlags aspects = |
| vk_format_aspects(ds_attachment->desc.format); |
| |
| /* Only clear once on the first subpass that uses the attachment */ |
| bool needs_depth_clear = |
| (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) && |
| state->tile_aligned_render_area && |
| state->job->first_subpass == ds_attachment->first_subpass && |
| ds_attachment->desc.loadOp == VK_ATTACHMENT_LOAD_OP_CLEAR && |
| !state->job->is_subpass_continue; |
| |
| bool needs_stencil_clear = |
| (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) && |
| state->tile_aligned_render_area && |
| state->job->first_subpass == ds_attachment->first_subpass && |
| ds_attachment->desc.stencilLoadOp == VK_ATTACHMENT_LOAD_OP_CLEAR && |
| !state->job->is_subpass_continue; |
| |
| /* Skip the last store if it is not required */ |
| bool needs_depth_store = |
| (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) && |
| (state->subpass_idx < ds_attachment->last_subpass || |
| ds_attachment->desc.storeOp == VK_ATTACHMENT_STORE_OP_STORE || |
| needs_depth_clear || |
| !state->job->is_subpass_finish); |
| |
| bool needs_stencil_store = |
| (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) && |
| (state->subpass_idx < ds_attachment->last_subpass || |
| ds_attachment->desc.stencilStoreOp == VK_ATTACHMENT_STORE_OP_STORE || |
| needs_stencil_clear || |
| !state->job->is_subpass_finish); |
| |
| /* GFXH-1461/GFXH-1689: The per-buffer store command's clear |
| * buffer bit is broken for depth/stencil. In addition, the |
| * clear packet's Z/S bit is broken, but the RTs bit ends up |
| * clearing Z/S. |
| * |
| * So if we have to emit a clear of depth or stencil we don't use |
| * per-buffer clears, not even for color, since we will have to emit |
| * a clear command for all tile buffers (including color) to handle |
| * the depth/stencil clears. |
| * |
| * Note that this bug is not reproduced in the simulator, where |
| * using the clear buffer bit in depth/stencil stores seems to work |
| * correctly. |
| */ |
| use_per_buffer_clear = !needs_stencil_clear && !needs_depth_clear; |
| bool needs_ds_store = needs_stencil_store || needs_depth_store; |
| if (needs_ds_store) { |
| uint32_t zs_buffer = v3dv_zs_buffer_from_aspect_bits(aspects); |
| cmd_buffer_render_pass_emit_store(cmd_buffer, cl, |
| ds_attachment_idx, layer, |
| zs_buffer, false); |
| has_stores = true; |
| } |
| } |
| |
| for (uint32_t i = 0; i < subpass->color_count; i++) { |
| uint32_t attachment_idx = subpass->color_attachments[i].attachment; |
| |
| if (attachment_idx == VK_ATTACHMENT_UNUSED) |
| continue; |
| |
| const struct v3dv_render_pass_attachment *attachment = |
| &state->pass->attachments[attachment_idx]; |
| |
| assert(state->job->first_subpass >= attachment->first_subpass); |
| assert(state->subpass_idx >= attachment->first_subpass); |
| assert(state->subpass_idx <= attachment->last_subpass); |
| |
| /* Only clear once on the first subpass that uses the attachment */ |
| bool needs_clear = |
| state->tile_aligned_render_area && |
| state->job->first_subpass == attachment->first_subpass && |
| attachment->desc.loadOp == VK_ATTACHMENT_LOAD_OP_CLEAR && |
| !state->job->is_subpass_continue; |
| |
| /* Skip the last store if it is not required */ |
| bool needs_store = |
| state->subpass_idx < attachment->last_subpass || |
| attachment->desc.storeOp == VK_ATTACHMENT_STORE_OP_STORE || |
| needs_clear || |
| !state->job->is_subpass_finish; |
| |
| if (needs_store) { |
| cmd_buffer_render_pass_emit_store(cmd_buffer, cl, |
| attachment_idx, layer, |
| RENDER_TARGET_0 + i, |
| needs_clear && use_per_buffer_clear); |
| has_stores = true; |
| } |
| } |
| |
| /* We always need to emit at least one dummy store */ |
| if (!has_stores) { |
| cl_emit(cl, STORE_TILE_BUFFER_GENERAL, store) { |
| store.buffer_to_store = NONE; |
| } |
| } |
| |
| /* If we have any depth/stencil clears we can't use the per-buffer clear |
| * bit and instead we have to emit a single clear of all tile buffers. |
| */ |
| if (!use_per_buffer_clear) { |
| cl_emit(cl, CLEAR_TILE_BUFFERS, clear) { |
| clear.clear_z_stencil_buffer = true; |
| clear.clear_all_render_targets = true; |
| } |
| } |
| } |
| |
| static void |
| cmd_buffer_render_pass_emit_per_tile_rcl(struct v3dv_cmd_buffer *cmd_buffer, |
| uint32_t layer) |
| { |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| /* Emit the generic list in our indirect state -- the rcl will just |
| * have pointers into it. |
| */ |
| struct v3dv_cl *cl = &job->indirect; |
| v3dv_cl_ensure_space(cl, 200, 1); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| struct v3dv_cl_reloc tile_list_start = v3dv_cl_get_address(cl); |
| |
| cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords); |
| |
| cmd_buffer_render_pass_emit_loads(cmd_buffer, cl, layer); |
| |
| /* The binner starts out writing tiles assuming that the initial mode |
| * is triangles, so make sure that's the case. |
| */ |
| cl_emit(cl, PRIM_LIST_FORMAT, fmt) { |
| fmt.primitive_type = LIST_TRIANGLES; |
| } |
| |
| cl_emit(cl, BRANCH_TO_IMPLICIT_TILE_LIST, branch); |
| |
| cmd_buffer_render_pass_emit_stores(cmd_buffer, cl, layer); |
| |
| cl_emit(cl, END_OF_TILE_MARKER, end); |
| |
| cl_emit(cl, RETURN_FROM_SUB_LIST, ret); |
| |
| cl_emit(&job->rcl, START_ADDRESS_OF_GENERIC_TILE_LIST, branch) { |
| branch.start = tile_list_start; |
| branch.end = v3dv_cl_get_address(cl); |
| } |
| } |
| |
| static void |
| cmd_buffer_emit_render_pass_layer_rcl(struct v3dv_cmd_buffer *cmd_buffer, |
| uint32_t layer) |
| { |
| const struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| |
| struct v3dv_job *job = cmd_buffer->state.job; |
| struct v3dv_cl *rcl = &job->rcl; |
| |
| /* If doing multicore binning, we would need to initialize each |
| * core's tile list here. |
| */ |
| const struct v3dv_frame_tiling *tiling = &job->frame_tiling; |
| const uint32_t tile_alloc_offset = |
| 64 * layer * tiling->draw_tiles_x * tiling->draw_tiles_y; |
| cl_emit(rcl, MULTICORE_RENDERING_TILE_LIST_SET_BASE, list) { |
| list.address = v3dv_cl_address(job->tile_alloc, tile_alloc_offset); |
| } |
| |
| cl_emit(rcl, MULTICORE_RENDERING_SUPERTILE_CFG, config) { |
| config.number_of_bin_tile_lists = 1; |
| config.total_frame_width_in_tiles = tiling->draw_tiles_x; |
| config.total_frame_height_in_tiles = tiling->draw_tiles_y; |
| |
| config.supertile_width_in_tiles = tiling->supertile_width; |
| config.supertile_height_in_tiles = tiling->supertile_height; |
| |
| config.total_frame_width_in_supertiles = |
| tiling->frame_width_in_supertiles; |
| config.total_frame_height_in_supertiles = |
| tiling->frame_height_in_supertiles; |
| } |
| |
| /* Start by clearing the tile buffer. */ |
| cl_emit(rcl, TILE_COORDINATES, coords) { |
| coords.tile_column_number = 0; |
| coords.tile_row_number = 0; |
| } |
| |
| /* Emit an initial clear of the tile buffers. This is necessary |
| * for any buffers that should be cleared (since clearing |
| * normally happens at the *end* of the generic tile list), but |
| * it's also nice to clear everything so the first tile doesn't |
| * inherit any contents from some previous frame. |
| * |
| * Also, implement the GFXH-1742 workaround. There's a race in |
| * the HW between the RCL updating the TLB's internal type/size |
| * and the spawning of the QPU instances using the TLB's current |
| * internal type/size. To make sure the QPUs get the right |
| * state, we need 1 dummy store in between internal type/size |
| * changes on V3D 3.x, and 2 dummy stores on 4.x. |
| */ |
| for (int i = 0; i < 2; i++) { |
| if (i > 0) |
| cl_emit(rcl, TILE_COORDINATES, coords); |
| cl_emit(rcl, END_OF_LOADS, end); |
| cl_emit(rcl, STORE_TILE_BUFFER_GENERAL, store) { |
| store.buffer_to_store = NONE; |
| } |
| if (i == 0 && cmd_buffer->state.tile_aligned_render_area) { |
| cl_emit(rcl, CLEAR_TILE_BUFFERS, clear) { |
| clear.clear_z_stencil_buffer = true; |
| clear.clear_all_render_targets = true; |
| } |
| } |
| cl_emit(rcl, END_OF_TILE_MARKER, end); |
| } |
| |
| cl_emit(rcl, FLUSH_VCD_CACHE, flush); |
| |
| cmd_buffer_render_pass_emit_per_tile_rcl(cmd_buffer, layer); |
| |
| uint32_t supertile_w_in_pixels = |
| tiling->tile_width * tiling->supertile_width; |
| uint32_t supertile_h_in_pixels = |
| tiling->tile_height * tiling->supertile_height; |
| const uint32_t min_x_supertile = |
| state->render_area.offset.x / supertile_w_in_pixels; |
| const uint32_t min_y_supertile = |
| state->render_area.offset.y / supertile_h_in_pixels; |
| |
| uint32_t max_render_x = state->render_area.offset.x; |
| if (state->render_area.extent.width > 0) |
| max_render_x += state->render_area.extent.width - 1; |
| uint32_t max_render_y = state->render_area.offset.y; |
| if (state->render_area.extent.height > 0) |
| max_render_y += state->render_area.extent.height - 1; |
| const uint32_t max_x_supertile = max_render_x / supertile_w_in_pixels; |
| const uint32_t max_y_supertile = max_render_y / supertile_h_in_pixels; |
| |
| for (int y = min_y_supertile; y <= max_y_supertile; y++) { |
| for (int x = min_x_supertile; x <= max_x_supertile; x++) { |
| cl_emit(rcl, SUPERTILE_COORDINATES, coords) { |
| coords.column_number_in_supertiles = x; |
| coords.row_number_in_supertiles = y; |
| } |
| } |
| } |
| } |
| |
| static void |
| set_rcl_early_z_config(struct v3dv_job *job, |
| bool *early_z_disable, |
| uint32_t *early_z_test_and_update_direction) |
| { |
| switch (job->first_ez_state) { |
| case VC5_EZ_UNDECIDED: |
| case VC5_EZ_LT_LE: |
| *early_z_disable = false; |
| *early_z_test_and_update_direction = EARLY_Z_DIRECTION_LT_LE; |
| break; |
| case VC5_EZ_GT_GE: |
| *early_z_disable = false; |
| *early_z_test_and_update_direction = EARLY_Z_DIRECTION_GT_GE; |
| break; |
| case VC5_EZ_DISABLED: |
| *early_z_disable = true; |
| break; |
| } |
| } |
| |
| static void |
| cmd_buffer_emit_render_pass_rcl(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| const struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| const struct v3dv_framebuffer *framebuffer = state->framebuffer; |
| |
| /* We can't emit the RCL until we have a framebuffer, which we may not have |
| * if we are recording a secondary command buffer. In that case, we will |
| * have to wait until vkCmdExecuteCommands is called from a primary command |
| * buffer. |
| */ |
| if (!framebuffer) { |
| assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY); |
| return; |
| } |
| |
| const struct v3dv_frame_tiling *tiling = &job->frame_tiling; |
| |
| const uint32_t fb_layers = framebuffer->layers; |
| v3dv_cl_ensure_space_with_branch(&job->rcl, 200 + |
| MAX2(fb_layers, 1) * 256 * |
| cl_packet_length(SUPERTILE_COORDINATES)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| assert(state->subpass_idx < state->pass->subpass_count); |
| const struct v3dv_subpass *subpass = |
| &state->pass->subpasses[state->subpass_idx]; |
| |
| struct v3dv_cl *rcl = &job->rcl; |
| |
| /* Comon config must be the first TILE_RENDERING_MODE_CFG and |
| * Z_STENCIL_CLEAR_VALUES must be last. The ones in between are optional |
| * updates to the previous HW state. |
| */ |
| const uint32_t ds_attachment_idx = subpass->ds_attachment.attachment; |
| |
| cl_emit(rcl, TILE_RENDERING_MODE_CFG_COMMON, config) { |
| config.image_width_pixels = framebuffer->width; |
| config.image_height_pixels = framebuffer->height; |
| config.number_of_render_targets = MAX2(subpass->color_count, 1); |
| config.multisample_mode_4x = false; /* FIXME */ |
| config.maximum_bpp_of_all_render_targets = tiling->internal_bpp; |
| |
| if (ds_attachment_idx != VK_ATTACHMENT_UNUSED) { |
| const struct v3dv_image_view *iview = |
| framebuffer->attachments[ds_attachment_idx]; |
| config.internal_depth_type = iview->internal_type; |
| } |
| |
| set_rcl_early_z_config(job, |
| &config.early_z_disable, |
| &config.early_z_test_and_update_direction); |
| } |
| |
| for (uint32_t i = 0; i < subpass->color_count; i++) { |
| uint32_t attachment_idx = subpass->color_attachments[i].attachment; |
| if (attachment_idx == VK_ATTACHMENT_UNUSED) |
| continue; |
| |
| struct v3dv_image_view *iview = |
| state->framebuffer->attachments[attachment_idx]; |
| |
| const struct v3dv_image *image = iview->image; |
| const struct v3d_resource_slice *slice = &image->slices[iview->base_level]; |
| |
| /* FIXME */ |
| assert(image->samples == VK_SAMPLE_COUNT_1_BIT); |
| |
| const uint32_t *clear_color = |
| &state->attachments[attachment_idx].clear_value.color[0]; |
| |
| uint32_t clear_pad = 0; |
| if (slice->tiling == VC5_TILING_UIF_NO_XOR || |
| slice->tiling == VC5_TILING_UIF_XOR) { |
| int uif_block_height = v3d_utile_height(image->cpp) * 2; |
| |
| uint32_t implicit_padded_height = |
| align(framebuffer->height, uif_block_height) / uif_block_height; |
| |
| if (slice->padded_height_of_output_image_in_uif_blocks - |
| implicit_padded_height >= 15) { |
| clear_pad = slice->padded_height_of_output_image_in_uif_blocks; |
| } |
| } |
| |
| cl_emit(rcl, TILE_RENDERING_MODE_CFG_CLEAR_COLORS_PART1, clear) { |
| clear.clear_color_low_32_bits = clear_color[0]; |
| clear.clear_color_next_24_bits = clear_color[1] & 0xffffff; |
| clear.render_target_number = i; |
| }; |
| |
| if (iview->internal_bpp >= V3D_INTERNAL_BPP_64) { |
| cl_emit(rcl, TILE_RENDERING_MODE_CFG_CLEAR_COLORS_PART2, clear) { |
| clear.clear_color_mid_low_32_bits = |
| ((clear_color[1] >> 24) | (clear_color[2] << 8)); |
| clear.clear_color_mid_high_24_bits = |
| ((clear_color[2] >> 24) | ((clear_color[3] & 0xffff) << 8)); |
| clear.render_target_number = i; |
| }; |
| } |
| |
| if (iview->internal_bpp >= V3D_INTERNAL_BPP_128 || clear_pad) { |
| cl_emit(rcl, TILE_RENDERING_MODE_CFG_CLEAR_COLORS_PART3, clear) { |
| clear.uif_padded_height_in_uif_blocks = clear_pad; |
| clear.clear_color_high_16_bits = clear_color[3] >> 16; |
| clear.render_target_number = i; |
| }; |
| } |
| } |
| |
| cl_emit(rcl, TILE_RENDERING_MODE_CFG_COLOR, rt) { |
| v3dv_render_pass_setup_render_target(cmd_buffer, 0, |
| &rt.render_target_0_internal_bpp, |
| &rt.render_target_0_internal_type, |
| &rt.render_target_0_clamp); |
| v3dv_render_pass_setup_render_target(cmd_buffer, 1, |
| &rt.render_target_1_internal_bpp, |
| &rt.render_target_1_internal_type, |
| &rt.render_target_1_clamp); |
| v3dv_render_pass_setup_render_target(cmd_buffer, 2, |
| &rt.render_target_2_internal_bpp, |
| &rt.render_target_2_internal_type, |
| &rt.render_target_2_clamp); |
| v3dv_render_pass_setup_render_target(cmd_buffer, 3, |
| &rt.render_target_3_internal_bpp, |
| &rt.render_target_3_internal_type, |
| &rt.render_target_3_clamp); |
| } |
| |
| /* Ends rendering mode config. */ |
| if (ds_attachment_idx != VK_ATTACHMENT_UNUSED) { |
| cl_emit(rcl, TILE_RENDERING_MODE_CFG_ZS_CLEAR_VALUES, clear) { |
| clear.z_clear_value = |
| state->attachments[ds_attachment_idx].clear_value.z; |
| clear.stencil_clear_value = |
| state->attachments[ds_attachment_idx].clear_value.s; |
| }; |
| } else { |
| cl_emit(rcl, TILE_RENDERING_MODE_CFG_ZS_CLEAR_VALUES, clear) { |
| clear.z_clear_value = 1.0f; |
| clear.stencil_clear_value = 0; |
| }; |
| } |
| |
| /* Always set initial block size before the first branch, which needs |
| * to match the value from binning mode config. |
| */ |
| cl_emit(rcl, TILE_LIST_INITIAL_BLOCK_SIZE, init) { |
| init.use_auto_chained_tile_lists = true; |
| init.size_of_first_block_in_chained_tile_lists = |
| TILE_ALLOCATION_BLOCK_SIZE_64B; |
| } |
| |
| for (int layer = 0; layer < MAX2(1, fb_layers); layer++) |
| cmd_buffer_emit_render_pass_layer_rcl(cmd_buffer, layer); |
| |
| cl_emit(rcl, END_OF_RENDERING, end); |
| } |
| |
| static void |
| cmd_buffer_emit_subpass_clears(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY); |
| |
| assert(cmd_buffer->state.pass); |
| assert(cmd_buffer->state.subpass_idx < cmd_buffer->state.pass->subpass_count); |
| const struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| const struct v3dv_render_pass *pass = state->pass; |
| const struct v3dv_subpass *subpass = &pass->subpasses[state->subpass_idx]; |
| |
| uint32_t att_count = 0; |
| VkClearAttachment atts[V3D_MAX_DRAW_BUFFERS + 1]; /* 4 color + D/S */ |
| for (uint32_t i = 0; i < subpass->color_count; i++) { |
| const uint32_t att_idx = subpass->color_attachments[i].attachment; |
| if (att_idx == VK_ATTACHMENT_UNUSED) |
| continue; |
| |
| struct v3dv_render_pass_attachment *att = &pass->attachments[att_idx]; |
| if (att->desc.loadOp != VK_ATTACHMENT_LOAD_OP_CLEAR) |
| continue; |
| |
| if (state->subpass_idx != att->first_subpass) |
| continue; |
| |
| atts[att_count].aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; |
| atts[att_count].colorAttachment = i; |
| atts[att_count].clearValue = state->attachments[att_idx].vk_clear_value; |
| att_count++; |
| } |
| |
| const uint32_t ds_att_idx = subpass->ds_attachment.attachment; |
| if (ds_att_idx != VK_ATTACHMENT_UNUSED) { |
| struct v3dv_render_pass_attachment *att = &pass->attachments[ds_att_idx]; |
| if (state->subpass_idx == att->first_subpass) { |
| VkImageAspectFlags aspects = vk_format_aspects(att->desc.format); |
| if (att->desc.loadOp != VK_ATTACHMENT_LOAD_OP_CLEAR) |
| aspects &= ~VK_IMAGE_ASPECT_DEPTH_BIT; |
| if (att->desc.stencilLoadOp != VK_ATTACHMENT_LOAD_OP_CLEAR) |
| aspects &= ~VK_IMAGE_ASPECT_STENCIL_BIT; |
| |
| atts[att_count].aspectMask = aspects; |
| atts[att_count].colorAttachment = 0; /* Ignored */ |
| atts[att_count].clearValue = |
| state->attachments[ds_att_idx].vk_clear_value; |
| att_count++; |
| } |
| } |
| |
| if (att_count == 0) |
| return; |
| |
| perf_debug("Render area doesn't match render pass granularity, falling back " |
| "to vkCmdClearAttachments for VK_ATTACHMENT_LOAD_OP_CLEAR.\n"); |
| |
| /* From the Vulkan 1.0 spec: |
| * |
| * "VK_ATTACHMENT_LOAD_OP_CLEAR specifies that the contents within the |
| * render area will be cleared to a uniform value, which is specified |
| * when a render pass instance is begun." |
| * |
| * So the clear is only constrained by the render area and not by pipeline |
| * state such as scissor or viewport, these are the semantics of |
| * vkCmdClearAttachments as well. |
| */ |
| VkCommandBuffer _cmd_buffer = v3dv_cmd_buffer_to_handle(cmd_buffer); |
| VkClearRect rect = { |
| .rect = state->render_area, |
| .baseArrayLayer = 0, |
| .layerCount = 1, |
| }; |
| v3dv_CmdClearAttachments(_cmd_buffer, att_count, atts, 1, &rect); |
| } |
| |
| static struct v3dv_job * |
| cmd_buffer_subpass_create_job(struct v3dv_cmd_buffer *cmd_buffer, |
| uint32_t subpass_idx, |
| enum v3dv_job_type type) |
| { |
| assert(type == V3DV_JOB_TYPE_GPU_CL || |
| type == V3DV_JOB_TYPE_GPU_CL_SECONDARY); |
| |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| assert(subpass_idx < state->pass->subpass_count); |
| |
| /* Starting a new job can trigger a finish of the current one, so don't |
| * change the command buffer state for the new job until we are done creating |
| * the new job. |
| */ |
| struct v3dv_job *job = |
| v3dv_cmd_buffer_start_job(cmd_buffer, subpass_idx, type); |
| if (!job) |
| return NULL; |
| |
| state->subpass_idx = subpass_idx; |
| |
| /* If we are starting a new job we need to setup binning. We only do this |
| * for V3DV_JOB_TYPE_GPU_CL jobs because V3DV_JOB_TYPE_GPU_CL_SECONDARY |
| * jobs are not submitted to the GPU directly, and are instead meant to be |
| * branched to from other V3DV_JOB_TYPE_GPU_CL jobs. |
| */ |
| if (type == V3DV_JOB_TYPE_GPU_CL && |
| job->first_subpass == state->subpass_idx) { |
| const struct v3dv_subpass *subpass = |
| &state->pass->subpasses[state->subpass_idx]; |
| |
| const struct v3dv_framebuffer *framebuffer = state->framebuffer; |
| |
| const uint8_t internal_bpp = |
| v3dv_framebuffer_compute_internal_bpp(framebuffer, subpass); |
| |
| v3dv_job_start_frame(job, |
| framebuffer->width, |
| framebuffer->height, |
| framebuffer->layers, |
| subpass->color_count, |
| internal_bpp); |
| |
| /* FIXME: we don't support input/resolve attachments yet */ |
| assert(subpass->input_count == 0); |
| assert(subpass->resolve_attachments == NULL); |
| } |
| |
| return job; |
| } |
| |
| struct v3dv_job * |
| v3dv_cmd_buffer_subpass_start(struct v3dv_cmd_buffer *cmd_buffer, |
| uint32_t subpass_idx) |
| { |
| assert(cmd_buffer->state.pass); |
| assert(subpass_idx < cmd_buffer->state.pass->subpass_count); |
| |
| struct v3dv_job *job = |
| cmd_buffer_subpass_create_job(cmd_buffer, subpass_idx, |
| V3DV_JOB_TYPE_GPU_CL); |
| if (!job) |
| return NULL; |
| |
| /* Check if our render area is aligned to tile boundaries. We have to do |
| * this in each subpass because the subset of attachments used can change |
| * and with that the tile size selected by the hardware can change too. |
| */ |
| cmd_buffer_update_tile_alignment(cmd_buffer); |
| |
| /* If we can't use TLB clears then we need to emit draw clears for any |
| * LOAD_OP_CLEAR attachments in this subpass now. |
| * |
| * Secondary command buffers don't start subpasses (and may not even have |
| * framebuffer state), so we only care about this in primaries. The only |
| * exception could be a secondary runnning inside a subpass that needs to |
| * record a meta operation (with its own render pass) that relies on |
| * attachment load clears, but we don't have any instances of that right |
| * now. |
| */ |
| if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY && |
| !cmd_buffer->state.tile_aligned_render_area) { |
| cmd_buffer_emit_subpass_clears(cmd_buffer); |
| } |
| |
| return job; |
| } |
| |
| struct v3dv_job * |
| v3dv_cmd_buffer_subpass_resume(struct v3dv_cmd_buffer *cmd_buffer, |
| uint32_t subpass_idx) |
| { |
| assert(cmd_buffer->state.pass); |
| assert(subpass_idx < cmd_buffer->state.pass->subpass_count); |
| |
| struct v3dv_job *job; |
| if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY) { |
| job = cmd_buffer_subpass_create_job(cmd_buffer, subpass_idx, |
| V3DV_JOB_TYPE_GPU_CL); |
| } else { |
| assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY); |
| job = cmd_buffer_subpass_create_job(cmd_buffer, subpass_idx, |
| V3DV_JOB_TYPE_GPU_CL_SECONDARY); |
| } |
| |
| if (!job) |
| return NULL; |
| |
| job->is_subpass_continue = true; |
| |
| return job; |
| } |
| |
| void |
| v3dv_cmd_buffer_subpass_finish(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| /* We can end up here without a job if the last command recorded into the |
| * subpass already finished the job (for example a pipeline barrier). In |
| * that case we miss to set the is_subpass_finish flag, but that is not |
| * required for proper behavior. |
| */ |
| struct v3dv_job *job = cmd_buffer->state.job; |
| if (job) |
| job->is_subpass_finish = true; |
| } |
| |
| void |
| v3dv_CmdEndRenderPass(VkCommandBuffer commandBuffer) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| |
| /* Finalize last subpass */ |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| assert(state->subpass_idx == state->pass->subpass_count - 1); |
| v3dv_cmd_buffer_subpass_finish(cmd_buffer); |
| v3dv_cmd_buffer_finish_job(cmd_buffer); |
| |
| /* We are no longer inside a render pass */ |
| state->framebuffer = NULL; |
| state->pass = NULL; |
| state->subpass_idx = -1; |
| } |
| |
| VkResult |
| v3dv_EndCommandBuffer(VkCommandBuffer commandBuffer) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| |
| if (cmd_buffer->state.oom) |
| return VK_ERROR_OUT_OF_HOST_MEMORY; |
| |
| /* Primaries should have ended any recording jobs by the time they hit |
| * vkEndRenderPass (if we are inside a render pass). Commands outside |
| * a render pass instance (for both primaries and secondaries) spawn |
| * complete jobs too. So the only case where we can get here without |
| * finishing a recording job is when we are recording a secondary |
| * inside a render pass. |
| */ |
| if (cmd_buffer->state.job) { |
| assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY && |
| cmd_buffer->state.pass); |
| v3dv_cmd_buffer_finish_job(cmd_buffer); |
| } |
| |
| cmd_buffer->status = V3DV_CMD_BUFFER_STATUS_EXECUTABLE; |
| |
| return VK_SUCCESS; |
| } |
| |
| static void |
| emit_occlusion_query(struct v3dv_cmd_buffer *cmd_buffer); |
| |
| static void |
| ensure_array_state(struct v3dv_cmd_buffer *cmd_buffer, |
| uint32_t slot_size, |
| uint32_t used_count, |
| uint32_t *alloc_count, |
| void **ptr); |
| |
| static void |
| cmd_buffer_copy_secondary_end_query_state(struct v3dv_cmd_buffer *primary, |
| struct v3dv_cmd_buffer *secondary) |
| { |
| struct v3dv_cmd_buffer_state *p_state = &primary->state; |
| struct v3dv_cmd_buffer_state *s_state = &secondary->state; |
| |
| const uint32_t total_state_count = |
| p_state->query.end.used_count + s_state->query.end.used_count; |
| ensure_array_state(primary, |
| sizeof(struct v3dv_end_query_cpu_job_info), |
| total_state_count, |
| &p_state->query.end.alloc_count, |
| (void **) &p_state->query.end.states); |
| v3dv_return_if_oom(primary, NULL); |
| |
| for (uint32_t i = 0; i < s_state->query.end.used_count; i++) { |
| const struct v3dv_end_query_cpu_job_info *s_qstate = |
| &secondary->state.query.end.states[i]; |
| |
| struct v3dv_end_query_cpu_job_info *p_qstate = |
| &p_state->query.end.states[p_state->query.end.used_count++]; |
| |
| p_qstate->pool = s_qstate->pool; |
| p_qstate->query = s_qstate->query; |
| } |
| } |
| |
| static void |
| clone_bo_list(struct v3dv_cmd_buffer *cmd_buffer, |
| struct list_head *dst, |
| struct list_head *src) |
| { |
| assert(cmd_buffer); |
| |
| list_inithead(dst); |
| list_for_each_entry(struct v3dv_bo, bo, src, list_link) { |
| struct v3dv_bo *clone_bo = |
| vk_alloc(&cmd_buffer->device->alloc, sizeof(struct v3dv_bo), 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); |
| if (!clone_bo) { |
| v3dv_flag_oom(cmd_buffer, NULL); |
| return; |
| } |
| |
| *clone_bo = *bo; |
| list_addtail(&clone_bo->list_link, dst); |
| } |
| } |
| |
| /* Clones a job for inclusion in the given command buffer. Note that this |
| * doesn't make a deep copy so the cloned job it doesn't own any resources. |
| * Useful when we need to have a job in more than one list, which happens |
| * for jobs recorded in secondary command buffers when we want to execute |
| * them in primaries. |
| */ |
| static void |
| job_clone_in_cmd_buffer(struct v3dv_job *job, |
| struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_job *clone_job = vk_alloc(&job->device->alloc, |
| sizeof(struct v3dv_job), 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); |
| if (!clone_job) { |
| v3dv_flag_oom(cmd_buffer, NULL); |
| return; |
| } |
| |
| /* Cloned jobs don't duplicate resources! */ |
| *clone_job = *job; |
| clone_job->is_clone = true; |
| clone_job->cmd_buffer = cmd_buffer; |
| list_addtail(&clone_job->list_link, &cmd_buffer->jobs); |
| |
| /* We need to regen the BO lists so that they point to the BO list in the |
| * cloned job. Otherwise functions like list_length() will loop forever. |
| */ |
| if (job->type == V3DV_JOB_TYPE_GPU_CL) { |
| clone_bo_list(cmd_buffer, &clone_job->bcl.bo_list, &job->bcl.bo_list); |
| clone_bo_list(cmd_buffer, &clone_job->rcl.bo_list, &job->rcl.bo_list); |
| clone_bo_list(cmd_buffer, &clone_job->indirect.bo_list, |
| &job->indirect.bo_list); |
| } |
| } |
| |
| static void |
| cmd_buffer_execute_inside_pass(struct v3dv_cmd_buffer *primary, |
| uint32_t cmd_buffer_count, |
| const VkCommandBuffer *cmd_buffers) |
| { |
| assert(primary->state.job); |
| |
| if (primary->state.dirty & V3DV_CMD_DIRTY_OCCLUSION_QUERY) |
| emit_occlusion_query(primary); |
| |
| for (uint32_t i = 0; i < cmd_buffer_count; i++) { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, secondary, cmd_buffers[i]); |
| |
| assert(secondary->usage_flags & |
| VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT); |
| |
| list_for_each_entry(struct v3dv_job, secondary_job, |
| &secondary->jobs, list_link) { |
| if (secondary_job->type == V3DV_JOB_TYPE_GPU_CL_SECONDARY) { |
| /* If the job is a CL, then we branch to it from the primary BCL. |
| * In this case the secondary's BCL is finished with a |
| * RETURN_FROM_SUB_LIST command to return back to the primary BCL |
| * once we are done executing it. |
| */ |
| assert(v3dv_cl_offset(&secondary_job->rcl) == 0); |
| assert(secondary_job->bcl.bo); |
| |
| /* Sanity check that secondary BCL ends with RETURN_FROM_SUB_LIST */ |
| STATIC_ASSERT(cl_packet_length(RETURN_FROM_SUB_LIST) == 1); |
| assert(v3dv_cl_offset(&secondary_job->bcl) >= 1); |
| assert(*(((uint8_t *)secondary_job->bcl.next) - 1) == |
| V3D42_RETURN_FROM_SUB_LIST_opcode); |
| |
| /* If we had to split the primary job while executing the secondary |
| * we will have to create a new one so we can emit the branch |
| * instruction. |
| * |
| * FIXME: in this case, maybe just copy the secondary BCL without |
| * the RETURN_FROM_SUB_LIST into the primary job to skip the |
| * branch? |
| */ |
| struct v3dv_job *primary_job = primary->state.job; |
| if (!primary_job) { |
| primary_job = |
| v3dv_cmd_buffer_subpass_resume(primary, |
| primary->state.subpass_idx); |
| } |
| |
| /* Make sure our primary job has all required BO references */ |
| set_foreach(secondary_job->bos, entry) { |
| struct v3dv_bo *bo = (struct v3dv_bo *)entry->key; |
| v3dv_job_add_bo(primary_job, bo); |
| } |
| |
| /* Emit the branch instruction */ |
| v3dv_cl_ensure_space_with_branch(&primary_job->bcl, |
| cl_packet_length(BRANCH_TO_SUB_LIST)); |
| v3dv_return_if_oom(primary, NULL); |
| |
| cl_emit(&primary_job->bcl, BRANCH_TO_SUB_LIST, branch) { |
| branch.address = v3dv_cl_address(secondary_job->bcl.bo, 0); |
| } |
| } else if (secondary_job->type == V3DV_JOB_TYPE_CPU_CLEAR_ATTACHMENTS) { |
| const struct v3dv_clear_attachments_cpu_job_info *info = |
| &secondary_job->cpu.clear_attachments; |
| v3dv_CmdClearAttachments(v3dv_cmd_buffer_to_handle(primary), |
| info->attachment_count, |
| info->attachments, |
| info->rect_count, |
| info->rects); |
| } else { |
| /* This is a regular job (CPU or GPU), so just finish the current |
| * primary job (if any) and then add the secondary job to the |
| * primary's job list right after it. |
| */ |
| v3dv_cmd_buffer_finish_job(primary); |
| job_clone_in_cmd_buffer(secondary_job, primary); |
| } |
| } |
| |
| /* If the secondary has recorded any vkCmdEndQuery commands, we need to |
| * copy this state to the primary so it is processed properly when the |
| * current primary job is finished. |
| */ |
| cmd_buffer_copy_secondary_end_query_state(primary, secondary); |
| } |
| } |
| |
| static void |
| cmd_buffer_execute_outside_pass(struct v3dv_cmd_buffer *primary, |
| uint32_t cmd_buffer_count, |
| const VkCommandBuffer *cmd_buffers) |
| { |
| for (uint32_t i = 0; i < cmd_buffer_count; i++) { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, secondary, cmd_buffers[i]); |
| |
| assert(!(secondary->usage_flags & |
| VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT)); |
| |
| /* Secondary command buffers that execute outside a render pass create |
| * complete jobs with an RCL and tile setup, so we simply want to merge |
| * their job list into the primary's. However, because they may be |
| * executed into multiple primaries at the same time and we only have a |
| * single list_link in each job, we can't just add then to the primary's |
| * job list and we instead have to clone them first. |
| * |
| * Alternatively, we could create a "execute secondary" CPU job that |
| * when executed in a queue, would submit all the jobs in the referenced |
| * secondary command buffer. However, this would raise some challenges |
| * to make it work with the implementation of wait threads in the queue |
| * which we use for event waits, for example. |
| */ |
| list_for_each_entry(struct v3dv_job, secondary_job, |
| &secondary->jobs, list_link) { |
| /* These can only happen inside a render pass */ |
| assert(secondary_job->type != V3DV_JOB_TYPE_CPU_CLEAR_ATTACHMENTS); |
| assert(secondary_job->type != V3DV_JOB_TYPE_GPU_CL_SECONDARY); |
| job_clone_in_cmd_buffer(secondary_job, primary); |
| } |
| } |
| } |
| |
| void |
| v3dv_CmdExecuteCommands(VkCommandBuffer commandBuffer, |
| uint32_t commandBufferCount, |
| const VkCommandBuffer *pCommandBuffers) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, primary, commandBuffer); |
| |
| if (primary->state.pass != NULL) { |
| cmd_buffer_execute_inside_pass(primary, |
| commandBufferCount, pCommandBuffers); |
| } else { |
| cmd_buffer_execute_outside_pass(primary, |
| commandBufferCount, pCommandBuffers); |
| } |
| } |
| |
| /* This goes though the list of possible dynamic states in the pipeline and, |
| * for those that are not configured as dynamic, copies relevant state into |
| * the command buffer. |
| */ |
| static void |
| cmd_buffer_bind_pipeline_static_state(struct v3dv_cmd_buffer *cmd_buffer, |
| const struct v3dv_dynamic_state *src) |
| { |
| struct v3dv_dynamic_state *dest = &cmd_buffer->state.dynamic; |
| uint32_t dynamic_mask = src->mask; |
| uint32_t dirty = 0; |
| |
| if (!(dynamic_mask & V3DV_DYNAMIC_VIEWPORT)) { |
| dest->viewport.count = src->viewport.count; |
| if (memcmp(&dest->viewport.viewports, &src->viewport.viewports, |
| src->viewport.count * sizeof(VkViewport))) { |
| typed_memcpy(dest->viewport.viewports, |
| src->viewport.viewports, |
| src->viewport.count); |
| typed_memcpy(dest->viewport.scale, src->viewport.scale, |
| src->viewport.count); |
| typed_memcpy(dest->viewport.translate, src->viewport.translate, |
| src->viewport.count); |
| dirty |= V3DV_CMD_DIRTY_VIEWPORT; |
| } |
| } |
| |
| if (!(dynamic_mask & V3DV_DYNAMIC_SCISSOR)) { |
| dest->scissor.count = src->scissor.count; |
| if (memcmp(&dest->scissor.scissors, &src->scissor.scissors, |
| src->scissor.count * sizeof(VkRect2D))) { |
| typed_memcpy(dest->scissor.scissors, |
| src->scissor.scissors, src->scissor.count); |
| dirty |= V3DV_CMD_DIRTY_SCISSOR; |
| } |
| } |
| |
| if (!(dynamic_mask & V3DV_DYNAMIC_STENCIL_COMPARE_MASK)) { |
| if (memcmp(&dest->stencil_compare_mask, &src->stencil_compare_mask, |
| sizeof(src->stencil_compare_mask))) { |
| dest->stencil_compare_mask = src->stencil_compare_mask; |
| dirty |= V3DV_CMD_DIRTY_STENCIL_COMPARE_MASK; |
| } |
| } |
| |
| if (!(dynamic_mask & V3DV_DYNAMIC_STENCIL_WRITE_MASK)) { |
| if (memcmp(&dest->stencil_write_mask, &src->stencil_write_mask, |
| sizeof(src->stencil_write_mask))) { |
| dest->stencil_write_mask = src->stencil_write_mask; |
| dirty |= V3DV_CMD_DIRTY_STENCIL_WRITE_MASK; |
| } |
| } |
| |
| if (!(dynamic_mask & V3DV_DYNAMIC_STENCIL_REFERENCE)) { |
| if (memcmp(&dest->stencil_reference, &src->stencil_reference, |
| sizeof(src->stencil_reference))) { |
| dest->stencil_reference = src->stencil_reference; |
| dirty |= V3DV_CMD_DIRTY_STENCIL_REFERENCE; |
| } |
| } |
| |
| if (!(dynamic_mask & V3DV_DYNAMIC_BLEND_CONSTANTS)) { |
| if (memcmp(dest->blend_constants, src->blend_constants, |
| sizeof(src->blend_constants))) { |
| memcpy(dest->blend_constants, src->blend_constants, |
| sizeof(src->blend_constants)); |
| dirty |= V3DV_CMD_DIRTY_BLEND_CONSTANTS; |
| } |
| } |
| |
| if (!(dynamic_mask & V3DV_DYNAMIC_DEPTH_BIAS)) { |
| if (memcmp(&dest->depth_bias, &src->depth_bias, |
| sizeof(src->depth_bias))) { |
| memcpy(&dest->depth_bias, &src->depth_bias, sizeof(src->depth_bias)); |
| dirty |= V3DV_CMD_DIRTY_DEPTH_BIAS; |
| } |
| } |
| |
| if (!(dynamic_mask & V3DV_DYNAMIC_LINE_WIDTH)) { |
| if (dest->line_width != src->line_width) { |
| dest->line_width = src->line_width; |
| dirty |= V3DV_CMD_DIRTY_LINE_WIDTH; |
| } |
| } |
| |
| cmd_buffer->state.dynamic.mask = dynamic_mask; |
| cmd_buffer->state.dirty |= dirty; |
| } |
| |
| static void |
| job_update_ez_state(struct v3dv_job *job, |
| struct v3dv_pipeline *pipeline, |
| struct v3dv_cmd_buffer_state *state) |
| { |
| /* If we don't have a depth attachment at all, disable */ |
| if (!state->pass) { |
| job->ez_state = VC5_EZ_DISABLED; |
| return; |
| } |
| |
| assert(state->subpass_idx < state->pass->subpass_count); |
| struct v3dv_subpass *subpass = &state->pass->subpasses[state->subpass_idx]; |
| if (subpass->ds_attachment.attachment == VK_ATTACHMENT_UNUSED) { |
| job->ez_state = VC5_EZ_DISABLED; |
| return; |
| } |
| |
| /* Otherwise, look at the curently bound pipeline state */ |
| switch (pipeline->ez_state) { |
| case VC5_EZ_UNDECIDED: |
| /* If the pipeline didn't pick a direction but didn't disable, then go |
| * along with the current EZ state. This allows EZ optimization for Z |
| * func == EQUAL or NEVER. |
| */ |
| break; |
| |
| case VC5_EZ_LT_LE: |
| case VC5_EZ_GT_GE: |
| /* If the pipeline picked a direction, then it needs to match the current |
| * direction if we've decided on one. |
| */ |
| if (job->ez_state == VC5_EZ_UNDECIDED) |
| job->ez_state = pipeline->ez_state; |
| else if (job->ez_state != pipeline->ez_state) |
| job->ez_state = VC5_EZ_DISABLED; |
| break; |
| |
| case VC5_EZ_DISABLED: |
| /* If the pipeline disables EZ because of a bad Z func or stencil |
| * operation, then we can't do any more EZ in this frame. |
| */ |
| job->ez_state = VC5_EZ_DISABLED; |
| break; |
| } |
| |
| /* If the FS writes Z, then it may update against the chosen EZ direction */ |
| if (pipeline->fs->current_variant->prog_data.fs->writes_z) |
| job->ez_state = VC5_EZ_DISABLED; |
| |
| if (job->first_ez_state == VC5_EZ_UNDECIDED && |
| job->ez_state != VC5_EZ_DISABLED) { |
| job->first_ez_state = job->ez_state; |
| } |
| } |
| |
| /* Note that the following poopulate methods doesn't do a detailed fill-up of |
| * the v3d_fs_key. Here we just fill-up cmd_buffer specific info. All info |
| * coming from the pipeline create info was alredy filled up when the pipeline |
| * was created |
| */ |
| static void |
| cmd_buffer_populate_v3d_key(struct v3d_key *key, |
| struct v3dv_cmd_buffer *cmd_buffer, |
| VkPipelineBindPoint pipeline_binding) |
| { |
| if (cmd_buffer->state.pipeline->combined_index_map != NULL) { |
| struct v3dv_descriptor_map *texture_map = &cmd_buffer->state.pipeline->texture_map; |
| struct v3dv_descriptor_map *sampler_map = &cmd_buffer->state.pipeline->sampler_map; |
| struct v3dv_descriptor_state *descriptor_state = |
| &cmd_buffer->state.descriptor_state[pipeline_binding]; |
| |
| hash_table_foreach(cmd_buffer->state.pipeline->combined_index_map, entry) { |
| uint32_t combined_idx = (uint32_t)(uintptr_t) (entry->data); |
| uint32_t combined_idx_key = |
| cmd_buffer->state.pipeline->combined_index_to_key_map[combined_idx]; |
| uint32_t texture_idx; |
| uint32_t sampler_idx; |
| |
| v3dv_pipeline_combined_index_key_unpack(combined_idx_key, |
| &texture_idx, &sampler_idx); |
| |
| struct v3dv_image_view *image_view = |
| v3dv_descriptor_map_get_image_view(descriptor_state, |
| texture_map, |
| cmd_buffer->state.pipeline->layout, |
| texture_idx); |
| |
| assert(image_view); |
| |
| const struct v3dv_sampler *sampler = NULL; |
| if (sampler_idx != V3DV_NO_SAMPLER_IDX) { |
| sampler = |
| v3dv_descriptor_map_get_sampler(descriptor_state, |
| sampler_map, |
| cmd_buffer->state.pipeline->layout, |
| sampler_idx); |
| assert(sampler); |
| } |
| |
| key->tex[combined_idx].return_size = |
| v3dv_get_tex_return_size(image_view->format, |
| sampler ? sampler->compare_enable : false); |
| |
| if (key->tex[combined_idx].return_size == 16) { |
| key->tex[combined_idx].return_channels = 2; |
| } else { |
| key->tex[combined_idx].return_channels = 4; |
| } |
| |
| /* Note: we don't need to do anything for the swizzle, as that is |
| * handled with the swizzle info at the Texture State, and the |
| * default values for key->tex[].swizzle were already filled up on |
| * the pipeline populate. |
| */ |
| } |
| } |
| } |
| |
| static void |
| update_fs_variant(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_shader_variant *variant; |
| struct v3dv_pipeline_stage *p_stage = cmd_buffer->state.pipeline->fs; |
| struct v3d_fs_key local_key; |
| |
| /* We start with a copy of the original pipeline key */ |
| memcpy(&local_key, &p_stage->key.fs, sizeof(struct v3d_fs_key)); |
| |
| cmd_buffer_populate_v3d_key(&local_key.base, cmd_buffer, |
| VK_PIPELINE_BIND_POINT_GRAPHICS); |
| |
| VkResult vk_result; |
| variant = v3dv_get_shader_variant(p_stage, &local_key.base, |
| sizeof(struct v3d_fs_key), |
| &cmd_buffer->device->alloc, |
| &vk_result); |
| /* At this point we are not creating a vulkan object to return to the |
| * API user, so we can't really return back a OOM error |
| */ |
| assert(variant); |
| assert(vk_result == VK_SUCCESS); |
| |
| p_stage->current_variant = variant; |
| } |
| |
| static void |
| update_vs_variant(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_shader_variant *variant; |
| struct v3dv_pipeline_stage *p_stage; |
| struct v3d_vs_key local_key; |
| VkResult vk_result; |
| |
| /* We start with a copy of the original pipeline key */ |
| p_stage = cmd_buffer->state.pipeline->vs; |
| memcpy(&local_key, &p_stage->key.vs, sizeof(struct v3d_vs_key)); |
| |
| cmd_buffer_populate_v3d_key(&local_key.base, cmd_buffer, |
| VK_PIPELINE_BIND_POINT_GRAPHICS); |
| |
| variant = v3dv_get_shader_variant(p_stage, &local_key.base, |
| sizeof(struct v3d_vs_key), |
| &cmd_buffer->device->alloc, |
| &vk_result); |
| /* At this point we are not creating a vulkan object to return to the |
| * API user, so we can't really return back a OOM error |
| */ |
| assert(variant); |
| assert(vk_result == VK_SUCCESS); |
| |
| p_stage->current_variant = variant; |
| |
| /* Now the vs_bin */ |
| p_stage = cmd_buffer->state.pipeline->vs_bin; |
| memcpy(&local_key, &p_stage->key.vs, sizeof(struct v3d_vs_key)); |
| |
| cmd_buffer_populate_v3d_key(&local_key.base, cmd_buffer, |
| VK_PIPELINE_BIND_POINT_GRAPHICS); |
| variant = v3dv_get_shader_variant(p_stage, &local_key.base, |
| sizeof(struct v3d_vs_key), |
| &cmd_buffer->device->alloc, |
| &vk_result); |
| |
| /* At this point we are not creating a vulkan object to return to the |
| * API user, so we can't really return back a OOM error |
| */ |
| assert(variant); |
| assert(vk_result == VK_SUCCESS); |
| |
| p_stage->current_variant = variant; |
| } |
| |
| static void |
| update_cs_variant(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_shader_variant *variant; |
| struct v3dv_pipeline_stage *p_stage = cmd_buffer->state.pipeline->cs; |
| struct v3d_key local_key; |
| |
| /* We start with a copy of the original pipeline key */ |
| memcpy(&local_key, &p_stage->key.base, sizeof(struct v3d_key)); |
| |
| cmd_buffer_populate_v3d_key(&local_key, cmd_buffer, |
| VK_PIPELINE_BIND_POINT_COMPUTE); |
| |
| VkResult result; |
| variant = v3dv_get_shader_variant(p_stage, &local_key, |
| sizeof(struct v3d_key), |
| &cmd_buffer->device->alloc, |
| &result); |
| /* At this point we are not creating a vulkan object to return to the |
| * API user, so we can't really return back a OOM error |
| */ |
| assert(variant); |
| assert(result == VK_SUCCESS); |
| |
| p_stage->current_variant = variant; |
| } |
| |
| /* |
| * Some updates on the cmd buffer requires also updates on the shader being |
| * compiled at the pipeline. The poster boy here are textures, as the compiler |
| * needs to do certain things depending on the texture format. So here we |
| * re-create the v3d_keys and update the variant. Note that internally the |
| * pipeline has a variant cache (hash table) to avoid unneeded compilations |
| * |
| */ |
| static void |
| update_pipeline_variants(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| assert(cmd_buffer->state.pipeline); |
| |
| if (v3dv_pipeline_get_binding_point(cmd_buffer->state.pipeline) == |
| VK_PIPELINE_BIND_POINT_GRAPHICS) { |
| update_fs_variant(cmd_buffer); |
| update_vs_variant(cmd_buffer); |
| } else { |
| update_cs_variant(cmd_buffer); |
| } |
| } |
| |
| static void |
| bind_graphics_pipeline(struct v3dv_cmd_buffer *cmd_buffer, |
| struct v3dv_pipeline *pipeline) |
| { |
| assert(pipeline && !(pipeline->active_stages & VK_SHADER_STAGE_COMPUTE_BIT)); |
| if (cmd_buffer->state.pipeline == pipeline) |
| return; |
| |
| |
| /* Enable always flush if we are blending to sRGB render targets. This |
| * fixes test failures in: |
| * dEQP-VK.pipeline.blend.format.r8g8b8a8_srgb.* |
| * |
| * FIXME: not sure why we need this. The tile buffer is always linear, with |
| * conversion from/to sRGB happening on tile load/store operations. This |
| * means that when we enable flushing the only difference is that we convert |
| * to sRGB on the store after each draw call and we convert from sRGB on the |
| * load before each draw call, but the blend happens in linear format in the |
| * tile buffer anyway, which is the same scenario as if we didn't flush. |
| */ |
| assert(pipeline->subpass); |
| if (pipeline->subpass->has_srgb_rt && pipeline->blend.enables) { |
| assert(cmd_buffer->state.job); |
| cmd_buffer->state.job->always_flush = true; |
| perf_debug("flushing draw calls for subpass %d because bound pipeline " |
| "uses sRGB blending\n", cmd_buffer->state.subpass_idx); |
| } |
| |
| cmd_buffer->state.pipeline = pipeline; |
| |
| cmd_buffer_bind_pipeline_static_state(cmd_buffer, &pipeline->dynamic_state); |
| |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_PIPELINE; |
| } |
| |
| static void |
| bind_compute_pipeline(struct v3dv_cmd_buffer *cmd_buffer, |
| struct v3dv_pipeline *pipeline) |
| { |
| assert(pipeline && pipeline->active_stages == VK_SHADER_STAGE_COMPUTE_BIT); |
| |
| if (cmd_buffer->state.pipeline == pipeline) |
| return; |
| |
| cmd_buffer->state.pipeline = pipeline; |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_PIPELINE; |
| } |
| |
| void |
| v3dv_CmdBindPipeline(VkCommandBuffer commandBuffer, |
| VkPipelineBindPoint pipelineBindPoint, |
| VkPipeline _pipeline) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| V3DV_FROM_HANDLE(v3dv_pipeline, pipeline, _pipeline); |
| |
| switch (pipelineBindPoint) { |
| case VK_PIPELINE_BIND_POINT_COMPUTE: |
| bind_compute_pipeline(cmd_buffer, pipeline); |
| break; |
| |
| case VK_PIPELINE_BIND_POINT_GRAPHICS: |
| bind_graphics_pipeline(cmd_buffer, pipeline); |
| break; |
| |
| default: |
| assert(!"invalid bind point"); |
| break; |
| } |
| } |
| |
| /* FIXME: C&P from radv. tu has similar code. Perhaps common place? */ |
| void |
| v3dv_viewport_compute_xform(const VkViewport *viewport, |
| float scale[3], |
| float translate[3]) |
| { |
| float x = viewport->x; |
| float y = viewport->y; |
| float half_width = 0.5f * viewport->width; |
| float half_height = 0.5f * viewport->height; |
| double n = viewport->minDepth; |
| double f = viewport->maxDepth; |
| |
| scale[0] = half_width; |
| translate[0] = half_width + x; |
| scale[1] = half_height; |
| translate[1] = half_height + y; |
| |
| scale[2] = (f - n); |
| translate[2] = n; |
| |
| /* It seems that if the scale is small enough the hardware won't clip |
| * correctly so we work around this my choosing the smallest scale that |
| * seems to work. |
| * |
| * This case is exercised by CTS: |
| * dEQP-VK.draw.inverted_depth_ranges.nodepthclamp_deltazero |
| */ |
| const float min_abs_scale = 0.000009f; |
| if (fabs(scale[2]) < min_abs_scale) |
| scale[2] = min_abs_scale * (scale[2] < 0 ? -1.0f : 1.0f); |
| } |
| |
| void |
| v3dv_CmdSetViewport(VkCommandBuffer commandBuffer, |
| uint32_t firstViewport, |
| uint32_t viewportCount, |
| const VkViewport *pViewports) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| const uint32_t total_count = firstViewport + viewportCount; |
| |
| assert(firstViewport < MAX_VIEWPORTS); |
| assert(total_count >= 1 && total_count <= MAX_VIEWPORTS); |
| |
| if (state->dynamic.viewport.count < total_count) |
| state->dynamic.viewport.count = total_count; |
| |
| if (!memcmp(state->dynamic.viewport.viewports + firstViewport, |
| pViewports, viewportCount * sizeof(*pViewports))) { |
| return; |
| } |
| |
| memcpy(state->dynamic.viewport.viewports + firstViewport, pViewports, |
| viewportCount * sizeof(*pViewports)); |
| |
| for (uint32_t i = firstViewport; i < total_count; i++) { |
| v3dv_viewport_compute_xform(&state->dynamic.viewport.viewports[i], |
| state->dynamic.viewport.scale[i], |
| state->dynamic.viewport.translate[i]); |
| } |
| |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_VIEWPORT; |
| } |
| |
| void |
| v3dv_CmdSetScissor(VkCommandBuffer commandBuffer, |
| uint32_t firstScissor, |
| uint32_t scissorCount, |
| const VkRect2D *pScissors) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| |
| assert(firstScissor < MAX_SCISSORS); |
| assert(firstScissor + scissorCount >= 1 && |
| firstScissor + scissorCount <= MAX_SCISSORS); |
| |
| if (state->dynamic.scissor.count < firstScissor + scissorCount) |
| state->dynamic.scissor.count = firstScissor + scissorCount; |
| |
| if (!memcmp(state->dynamic.scissor.scissors + firstScissor, |
| pScissors, scissorCount * sizeof(*pScissors))) { |
| return; |
| } |
| |
| memcpy(state->dynamic.scissor.scissors + firstScissor, pScissors, |
| scissorCount * sizeof(*pScissors)); |
| |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_SCISSOR; |
| } |
| |
| static void |
| emit_scissor(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| if (cmd_buffer->state.dynamic.viewport.count == 0) |
| return; |
| |
| struct v3dv_dynamic_state *dynamic = &cmd_buffer->state.dynamic; |
| |
| /* FIXME: right now we only support one viewport. viewporst[0] would work |
| * now, but would need to change if we allow multiple viewports. |
| */ |
| float *vptranslate = dynamic->viewport.translate[0]; |
| float *vpscale = dynamic->viewport.scale[0]; |
| |
| float vp_minx = -fabsf(vpscale[0]) + vptranslate[0]; |
| float vp_maxx = fabsf(vpscale[0]) + vptranslate[0]; |
| float vp_miny = -fabsf(vpscale[1]) + vptranslate[1]; |
| float vp_maxy = fabsf(vpscale[1]) + vptranslate[1]; |
| |
| /* Quoting from v3dx_emit: |
| * "Clip to the scissor if it's enabled, but still clip to the |
| * drawable regardless since that controls where the binner |
| * tries to put things. |
| * |
| * Additionally, always clip the rendering to the viewport, |
| * since the hardware does guardband clipping, meaning |
| * primitives would rasterize outside of the view volume." |
| */ |
| uint32_t minx, miny, maxx, maxy; |
| |
| /* From the Vulkan spec: |
| * |
| * "The application must ensure (using scissor if necessary) that all |
| * rendering is contained within the render area. The render area must be |
| * contained within the framebuffer dimensions." |
| * |
| * So it is the application's responsibility to ensure this. Still, we can |
| * help by automatically restricting the scissor rect to the render area. |
| */ |
| minx = MAX2(vp_minx, cmd_buffer->state.render_area.offset.x); |
| miny = MAX2(vp_miny, cmd_buffer->state.render_area.offset.y); |
| maxx = MIN2(vp_maxx, cmd_buffer->state.render_area.offset.x + |
| cmd_buffer->state.render_area.extent.width); |
| maxy = MIN2(vp_maxy, cmd_buffer->state.render_area.offset.y + |
| cmd_buffer->state.render_area.extent.height); |
| |
| minx = vp_minx; |
| miny = vp_miny; |
| maxx = vp_maxx; |
| maxy = vp_maxy; |
| |
| /* Clip against user provided scissor if needed. |
| * |
| * FIXME: right now we only allow one scissor. Below would need to be |
| * updated if we support more |
| */ |
| if (dynamic->scissor.count > 0) { |
| VkRect2D *scissor = &dynamic->scissor.scissors[0]; |
| minx = MAX2(minx, scissor->offset.x); |
| miny = MAX2(miny, scissor->offset.y); |
| maxx = MIN2(maxx, scissor->offset.x + scissor->extent.width); |
| maxy = MIN2(maxy, scissor->offset.y + scissor->extent.height); |
| } |
| |
| /* If the scissor is outside the viewport area we end up with |
| * min{x,y} > max{x,y}. |
| */ |
| if (minx > maxx) |
| maxx = minx; |
| if (miny > maxy) |
| maxy = miny; |
| |
| cmd_buffer->state.clip_window.offset.x = minx; |
| cmd_buffer->state.clip_window.offset.y = miny; |
| cmd_buffer->state.clip_window.extent.width = maxx - minx; |
| cmd_buffer->state.clip_window.extent.height = maxy - miny; |
| |
| emit_clip_window(cmd_buffer->state.job, &cmd_buffer->state.clip_window); |
| |
| cmd_buffer->state.dirty &= ~V3DV_CMD_DIRTY_SCISSOR; |
| } |
| |
| static void |
| emit_viewport(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_dynamic_state *dynamic = &cmd_buffer->state.dynamic; |
| /* FIXME: right now we only support one viewport. viewporst[0] would work |
| * now, would need to change if we allow multiple viewports |
| */ |
| float *vptranslate = dynamic->viewport.translate[0]; |
| float *vpscale = dynamic->viewport.scale[0]; |
| |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| const uint32_t required_cl_size = |
| cl_packet_length(CLIPPER_XY_SCALING) + |
| cl_packet_length(CLIPPER_Z_SCALE_AND_OFFSET) + |
| cl_packet_length(CLIPPER_Z_MIN_MAX_CLIPPING_PLANES) + |
| cl_packet_length(VIEWPORT_OFFSET); |
| v3dv_cl_ensure_space_with_branch(&job->bcl, required_cl_size); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit(&job->bcl, CLIPPER_XY_SCALING, clip) { |
| clip.viewport_half_width_in_1_256th_of_pixel = vpscale[0] * 256.0f; |
| clip.viewport_half_height_in_1_256th_of_pixel = vpscale[1] * 256.0f; |
| } |
| |
| cl_emit(&job->bcl, CLIPPER_Z_SCALE_AND_OFFSET, clip) { |
| clip.viewport_z_offset_zc_to_zs = vptranslate[2]; |
| clip.viewport_z_scale_zc_to_zs = vpscale[2]; |
| } |
| cl_emit(&job->bcl, CLIPPER_Z_MIN_MAX_CLIPPING_PLANES, clip) { |
| /* Vulkan's Z NDC is [0..1], unlile OpenGL which is [-1, 1] */ |
| float z1 = vptranslate[2]; |
| float z2 = vptranslate[2] + vpscale[2]; |
| clip.minimum_zw = MIN2(z1, z2); |
| clip.maximum_zw = MAX2(z1, z2); |
| } |
| |
| cl_emit(&job->bcl, VIEWPORT_OFFSET, vp) { |
| vp.viewport_centre_x_coordinate = vptranslate[0]; |
| vp.viewport_centre_y_coordinate = vptranslate[1]; |
| } |
| |
| cmd_buffer->state.dirty &= ~V3DV_CMD_DIRTY_VIEWPORT; |
| } |
| |
| static void |
| emit_stencil(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| struct v3dv_pipeline *pipeline = cmd_buffer->state.pipeline; |
| struct v3dv_dynamic_state *dynamic_state = &cmd_buffer->state.dynamic; |
| |
| const uint32_t dynamic_stencil_states = V3DV_DYNAMIC_STENCIL_COMPARE_MASK | |
| V3DV_DYNAMIC_STENCIL_WRITE_MASK | |
| V3DV_DYNAMIC_STENCIL_REFERENCE; |
| |
| v3dv_cl_ensure_space_with_branch(&job->bcl, |
| 2 * cl_packet_length(STENCIL_CFG)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| bool emitted_stencil = false; |
| for (uint32_t i = 0; i < 2; i++) { |
| if (pipeline->emit_stencil_cfg[i]) { |
| if (dynamic_state->mask & dynamic_stencil_states) { |
| cl_emit_with_prepacked(&job->bcl, STENCIL_CFG, |
| pipeline->stencil_cfg[i], config) { |
| if (dynamic_state->mask & V3DV_DYNAMIC_STENCIL_COMPARE_MASK) { |
| config.stencil_test_mask = |
| i == 0 ? dynamic_state->stencil_compare_mask.front : |
| dynamic_state->stencil_compare_mask.back; |
| } |
| if (dynamic_state->mask & V3DV_DYNAMIC_STENCIL_WRITE_MASK) { |
| config.stencil_write_mask = |
| i == 0 ? dynamic_state->stencil_write_mask.front : |
| dynamic_state->stencil_write_mask.back; |
| } |
| if (dynamic_state->mask & V3DV_DYNAMIC_STENCIL_REFERENCE) { |
| config.stencil_ref_value = |
| i == 0 ? dynamic_state->stencil_reference.front : |
| dynamic_state->stencil_reference.back; |
| } |
| } |
| } else { |
| cl_emit_prepacked(&job->bcl, &pipeline->stencil_cfg[i]); |
| } |
| |
| emitted_stencil = true; |
| } |
| } |
| |
| if (emitted_stencil) { |
| const uint32_t dynamic_stencil_dirty_flags = |
| V3DV_CMD_DIRTY_STENCIL_COMPARE_MASK | |
| V3DV_CMD_DIRTY_STENCIL_WRITE_MASK | |
| V3DV_CMD_DIRTY_STENCIL_REFERENCE; |
| cmd_buffer->state.dirty &= ~dynamic_stencil_dirty_flags; |
| } |
| } |
| |
| static void |
| emit_depth_bias(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_pipeline *pipeline = cmd_buffer->state.pipeline; |
| assert(pipeline); |
| |
| if (!pipeline->depth_bias.enabled) |
| return; |
| |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(DEPTH_OFFSET)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| struct v3dv_dynamic_state *dynamic = &cmd_buffer->state.dynamic; |
| cl_emit(&job->bcl, DEPTH_OFFSET, bias) { |
| bias.depth_offset_factor = dynamic->depth_bias.slope_factor; |
| bias.depth_offset_units = dynamic->depth_bias.constant_factor; |
| if (pipeline->depth_bias.is_z16) |
| bias.depth_offset_units *= 256.0f; |
| } |
| |
| cmd_buffer->state.dirty &= ~V3DV_CMD_DIRTY_DEPTH_BIAS; |
| } |
| |
| static void |
| emit_line_width(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(LINE_WIDTH)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit(&job->bcl, LINE_WIDTH, line) { |
| line.line_width = cmd_buffer->state.dynamic.line_width; |
| } |
| |
| cmd_buffer->state.dirty &= ~V3DV_CMD_DIRTY_LINE_WIDTH; |
| } |
| |
| static void |
| emit_blend(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| struct v3dv_pipeline *pipeline = cmd_buffer->state.pipeline; |
| assert(pipeline); |
| |
| const uint32_t blend_packets_size = |
| cl_packet_length(BLEND_ENABLES) + |
| cl_packet_length(BLEND_CONSTANT_COLOR) + |
| cl_packet_length(BLEND_CFG) * V3D_MAX_DRAW_BUFFERS + |
| cl_packet_length(COLOR_WRITE_MASKS); |
| |
| v3dv_cl_ensure_space_with_branch(&job->bcl, blend_packets_size); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| if (cmd_buffer->state.dirty & V3DV_CMD_DIRTY_PIPELINE) { |
| if (pipeline->blend.enables) { |
| cl_emit(&job->bcl, BLEND_ENABLES, enables) { |
| enables.mask = pipeline->blend.enables; |
| } |
| } |
| |
| for (uint32_t i = 0; i < V3D_MAX_DRAW_BUFFERS; i++) { |
| if (pipeline->blend.enables & (1 << i)) |
| cl_emit_prepacked(&job->bcl, &pipeline->blend.cfg[i]); |
| } |
| |
| cl_emit(&job->bcl, COLOR_WRITE_MASKS, mask) { |
| mask.mask = pipeline->blend.color_write_masks; |
| } |
| } |
| |
| if (pipeline->blend.needs_color_constants && |
| cmd_buffer->state.dirty & V3DV_CMD_DIRTY_BLEND_CONSTANTS) { |
| struct v3dv_dynamic_state *dynamic = &cmd_buffer->state.dynamic; |
| cl_emit(&job->bcl, BLEND_CONSTANT_COLOR, color) { |
| color.red_f16 = _mesa_float_to_half(dynamic->blend_constants[0]); |
| color.green_f16 = _mesa_float_to_half(dynamic->blend_constants[1]); |
| color.blue_f16 = _mesa_float_to_half(dynamic->blend_constants[2]); |
| color.alpha_f16 = _mesa_float_to_half(dynamic->blend_constants[3]); |
| } |
| cmd_buffer->state.dirty &= ~V3DV_CMD_DIRTY_BLEND_CONSTANTS; |
| } |
| } |
| |
| static void |
| emit_flat_shade_flags(struct v3dv_job *job, |
| int varying_offset, |
| uint32_t varyings, |
| enum V3DX(Varying_Flags_Action) lower, |
| enum V3DX(Varying_Flags_Action) higher) |
| { |
| v3dv_cl_ensure_space_with_branch(&job->bcl, |
| cl_packet_length(FLAT_SHADE_FLAGS)); |
| v3dv_return_if_oom(NULL, job); |
| |
| cl_emit(&job->bcl, FLAT_SHADE_FLAGS, flags) { |
| flags.varying_offset_v0 = varying_offset; |
| flags.flat_shade_flags_for_varyings_v024 = varyings; |
| flags.action_for_flat_shade_flags_of_lower_numbered_varyings = lower; |
| flags.action_for_flat_shade_flags_of_higher_numbered_varyings = higher; |
| } |
| } |
| |
| static void |
| emit_noperspective_flags(struct v3dv_job *job, |
| int varying_offset, |
| uint32_t varyings, |
| enum V3DX(Varying_Flags_Action) lower, |
| enum V3DX(Varying_Flags_Action) higher) |
| { |
| v3dv_cl_ensure_space_with_branch(&job->bcl, |
| cl_packet_length(NON_PERSPECTIVE_FLAGS)); |
| v3dv_return_if_oom(NULL, job); |
| |
| cl_emit(&job->bcl, NON_PERSPECTIVE_FLAGS, flags) { |
| flags.varying_offset_v0 = varying_offset; |
| flags.non_perspective_flags_for_varyings_v024 = varyings; |
| flags.action_for_non_perspective_flags_of_lower_numbered_varyings = lower; |
| flags.action_for_non_perspective_flags_of_higher_numbered_varyings = higher; |
| } |
| } |
| |
| static void |
| emit_centroid_flags(struct v3dv_job *job, |
| int varying_offset, |
| uint32_t varyings, |
| enum V3DX(Varying_Flags_Action) lower, |
| enum V3DX(Varying_Flags_Action) higher) |
| { |
| v3dv_cl_ensure_space_with_branch(&job->bcl, |
| cl_packet_length(CENTROID_FLAGS)); |
| v3dv_return_if_oom(NULL, job); |
| |
| cl_emit(&job->bcl, CENTROID_FLAGS, flags) { |
| flags.varying_offset_v0 = varying_offset; |
| flags.centroid_flags_for_varyings_v024 = varyings; |
| flags.action_for_centroid_flags_of_lower_numbered_varyings = lower; |
| flags.action_for_centroid_flags_of_higher_numbered_varyings = higher; |
| } |
| } |
| |
| static bool |
| emit_varying_flags(struct v3dv_job *job, |
| uint32_t num_flags, |
| const uint32_t *flags, |
| void (*flag_emit_callback)(struct v3dv_job *job, |
| int varying_offset, |
| uint32_t flags, |
| enum V3DX(Varying_Flags_Action) lower, |
| enum V3DX(Varying_Flags_Action) higher)) |
| { |
| bool emitted_any = false; |
| for (int i = 0; i < num_flags; i++) { |
| if (!flags[i]) |
| continue; |
| |
| if (emitted_any) { |
| flag_emit_callback(job, i, flags[i], |
| V3D_VARYING_FLAGS_ACTION_UNCHANGED, |
| V3D_VARYING_FLAGS_ACTION_UNCHANGED); |
| } else if (i == 0) { |
| flag_emit_callback(job, i, flags[i], |
| V3D_VARYING_FLAGS_ACTION_UNCHANGED, |
| V3D_VARYING_FLAGS_ACTION_ZEROED); |
| } else { |
| flag_emit_callback(job, i, flags[i], |
| V3D_VARYING_FLAGS_ACTION_ZEROED, |
| V3D_VARYING_FLAGS_ACTION_ZEROED); |
| } |
| |
| emitted_any = true; |
| } |
| |
| return emitted_any; |
| } |
| |
| static void |
| emit_varyings_state(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_job *job = cmd_buffer->state.job; |
| struct v3dv_pipeline *pipeline = cmd_buffer->state.pipeline; |
| |
| struct v3d_fs_prog_data *prog_data_fs = |
| pipeline->fs->current_variant->prog_data.fs; |
| |
| const uint32_t num_flags = |
| ARRAY_SIZE(prog_data_fs->flat_shade_flags); |
| const uint32_t *flat_shade_flags = prog_data_fs->flat_shade_flags; |
| const uint32_t *noperspective_flags = prog_data_fs->noperspective_flags; |
| const uint32_t *centroid_flags = prog_data_fs->centroid_flags; |
| |
| if (!emit_varying_flags(job, num_flags, flat_shade_flags, |
| emit_flat_shade_flags)) { |
| v3dv_cl_ensure_space_with_branch( |
| &job->bcl, cl_packet_length(ZERO_ALL_FLAT_SHADE_FLAGS)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit(&job->bcl, ZERO_ALL_FLAT_SHADE_FLAGS, flags); |
| } |
| |
| if (!emit_varying_flags(job, num_flags, noperspective_flags, |
| emit_noperspective_flags)) { |
| v3dv_cl_ensure_space_with_branch( |
| &job->bcl, cl_packet_length(ZERO_ALL_NON_PERSPECTIVE_FLAGS)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit(&job->bcl, ZERO_ALL_NON_PERSPECTIVE_FLAGS, flags); |
| } |
| |
| if (!emit_varying_flags(job, num_flags, centroid_flags, |
| emit_centroid_flags)) { |
| v3dv_cl_ensure_space_with_branch( |
| &job->bcl, cl_packet_length(ZERO_ALL_CENTROID_FLAGS)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit(&job->bcl, ZERO_ALL_CENTROID_FLAGS, flags); |
| } |
| } |
| |
| static void |
| emit_configuration_bits(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| struct v3dv_pipeline *pipeline = cmd_buffer->state.pipeline; |
| assert(pipeline); |
| |
| job_update_ez_state(job, pipeline, &cmd_buffer->state); |
| |
| v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(CFG_BITS)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit_with_prepacked(&job->bcl, CFG_BITS, pipeline->cfg_bits, config) { |
| config.early_z_updates_enable = job->ez_state != VC5_EZ_DISABLED; |
| config.early_z_enable = config.early_z_updates_enable; |
| } |
| } |
| |
| static void |
| emit_gl_shader_state(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| struct v3dv_pipeline *pipeline = state->pipeline; |
| assert(pipeline); |
| |
| /* Upload the uniforms to the indirect CL first */ |
| struct v3dv_cl_reloc fs_uniforms = |
| v3dv_write_uniforms(cmd_buffer, pipeline->fs); |
| |
| struct v3dv_cl_reloc vs_uniforms = |
| v3dv_write_uniforms(cmd_buffer, pipeline->vs); |
| |
| struct v3dv_cl_reloc vs_bin_uniforms = |
| v3dv_write_uniforms(cmd_buffer, pipeline->vs_bin); |
| |
| /* Update the cache dirty flag based on the shader progs data */ |
| job->tmu_dirty_rcl |= pipeline->vs_bin->current_variant->prog_data.vs->base.tmu_dirty_rcl; |
| job->tmu_dirty_rcl |= pipeline->vs->current_variant->prog_data.vs->base.tmu_dirty_rcl; |
| job->tmu_dirty_rcl |= pipeline->fs->current_variant->prog_data.fs->base.tmu_dirty_rcl; |
| |
| /* See GFXH-930 workaround below */ |
| uint32_t num_elements_to_emit = MAX2(pipeline->va_count, 1); |
| |
| uint32_t shader_rec_offset = |
| v3dv_cl_ensure_space(&job->indirect, |
| cl_packet_length(GL_SHADER_STATE_RECORD) + |
| num_elements_to_emit * |
| cl_packet_length(GL_SHADER_STATE_ATTRIBUTE_RECORD), |
| 32); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit_with_prepacked(&job->indirect, GL_SHADER_STATE_RECORD, |
| pipeline->shader_state_record, shader) { |
| |
| /* FIXME: we are setting this values here and during the |
| * prepacking. This is because both cl_emit_with_prepacked and v3dv_pack |
| * asserts for minimum values of these. It would be good to get |
| * v3dv_pack to assert on the final value if possible |
| */ |
| shader.min_coord_shader_input_segments_required_in_play = |
| pipeline->vpm_cfg_bin.As; |
| shader.min_vertex_shader_input_segments_required_in_play = |
| pipeline->vpm_cfg.As; |
| |
| shader.coordinate_shader_code_address = |
| v3dv_cl_address(pipeline->vs_bin->current_variant->assembly_bo, 0); |
| shader.vertex_shader_code_address = |
| v3dv_cl_address(pipeline->vs->current_variant->assembly_bo, 0); |
| shader.fragment_shader_code_address = |
| v3dv_cl_address(pipeline->fs->current_variant->assembly_bo, 0); |
| |
| shader.coordinate_shader_uniforms_address = vs_bin_uniforms; |
| shader.vertex_shader_uniforms_address = vs_uniforms; |
| shader.fragment_shader_uniforms_address = fs_uniforms; |
| |
| shader.address_of_default_attribute_values = |
| v3dv_cl_address(pipeline->default_attribute_values, 0); |
| } |
| |
| /* Upload vertex element attributes (SHADER_STATE_ATTRIBUTE_RECORD) */ |
| bool cs_loaded_any = false; |
| const uint32_t packet_length = |
| cl_packet_length(GL_SHADER_STATE_ATTRIBUTE_RECORD); |
| |
| for (uint32_t i = 0; i < pipeline->va_count; i++) { |
| uint32_t binding = pipeline->va[i].binding; |
| uint32_t location = pipeline->va[i].driver_location; |
| |
| struct v3dv_vertex_binding *c_vb = &cmd_buffer->state.vertex_bindings[binding]; |
| |
| struct v3d_vs_prog_data *prog_data_vs = |
| pipeline->vs->current_variant->prog_data.vs; |
| |
| struct v3d_vs_prog_data *prog_data_vs_bin = |
| pipeline->vs_bin->current_variant->prog_data.vs; |
| |
| cl_emit_with_prepacked(&job->indirect, GL_SHADER_STATE_ATTRIBUTE_RECORD, |
| &pipeline->vertex_attrs[i * packet_length], attr) { |
| |
| assert(c_vb->buffer->mem->bo); |
| attr.address = v3dv_cl_address(c_vb->buffer->mem->bo, |
| c_vb->buffer->mem_offset + |
| pipeline->va[i].offset + |
| c_vb->offset); |
| |
| attr.number_of_values_read_by_coordinate_shader = |
| prog_data_vs_bin->vattr_sizes[location]; |
| attr.number_of_values_read_by_vertex_shader = |
| prog_data_vs->vattr_sizes[location]; |
| |
| /* GFXH-930: At least one attribute must be enabled and read by CS |
| * and VS. If we have attributes being consumed by the VS but not |
| * the CS, then set up a dummy load of the last attribute into the |
| * CS's VPM inputs. (Since CS is just dead-code-elimination compared |
| * to VS, we can't have CS loading but not VS). |
| */ |
| if (prog_data_vs->vattr_sizes[location]) |
| cs_loaded_any = true; |
| |
| if (binding == pipeline->va_count - 1 && !cs_loaded_any) { |
| attr.number_of_values_read_by_coordinate_shader = 1; |
| } |
| |
| attr.maximum_index = 0xffffff; |
| } |
| } |
| |
| if (pipeline->va_count == 0) { |
| /* GFXH-930: At least one attribute must be enabled and read |
| * by CS and VS. If we have no attributes being consumed by |
| * the shader, set up a dummy to be loaded into the VPM. |
| */ |
| cl_emit(&job->indirect, GL_SHADER_STATE_ATTRIBUTE_RECORD, attr) { |
| /* Valid address of data whose value will be unused. */ |
| attr.address = v3dv_cl_address(job->indirect.bo, 0); |
| |
| attr.type = ATTRIBUTE_FLOAT; |
| attr.stride = 0; |
| attr.vec_size = 1; |
| |
| attr.number_of_values_read_by_coordinate_shader = 1; |
| attr.number_of_values_read_by_vertex_shader = 1; |
| } |
| } |
| |
| if (cmd_buffer->state.dirty & V3DV_CMD_DIRTY_PIPELINE) { |
| v3dv_cl_ensure_space_with_branch(&job->bcl, |
| sizeof(pipeline->vcm_cache_size)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit_prepacked(&job->bcl, &pipeline->vcm_cache_size); |
| } |
| |
| v3dv_cl_ensure_space_with_branch(&job->bcl, |
| cl_packet_length(GL_SHADER_STATE)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit(&job->bcl, GL_SHADER_STATE, state) { |
| state.address = v3dv_cl_address(job->indirect.bo, |
| shader_rec_offset); |
| state.number_of_attribute_arrays = num_elements_to_emit; |
| } |
| |
| cmd_buffer->state.dirty &= ~(V3DV_CMD_DIRTY_VERTEX_BUFFER | |
| V3DV_CMD_DIRTY_DESCRIPTOR_SETS | |
| V3DV_CMD_DIRTY_PUSH_CONSTANTS); |
| } |
| |
| static void |
| emit_occlusion_query(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| v3dv_cl_ensure_space_with_branch(&job->bcl, |
| cl_packet_length(OCCLUSION_QUERY_COUNTER)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit(&job->bcl, OCCLUSION_QUERY_COUNTER, counter) { |
| if (cmd_buffer->state.query.active_query) { |
| counter.address = |
| v3dv_cl_address(cmd_buffer->state.query.active_query, 0); |
| } |
| } |
| |
| cmd_buffer->state.dirty &= ~V3DV_CMD_DIRTY_OCCLUSION_QUERY; |
| } |
| |
| /* This stores command buffer state that we might be about to stomp for |
| * a meta operation. |
| */ |
| void |
| v3dv_cmd_buffer_meta_state_push(struct v3dv_cmd_buffer *cmd_buffer, |
| bool push_descriptor_state) |
| { |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| |
| if (state->subpass_idx != -1) { |
| state->meta.subpass_idx = state->subpass_idx; |
| state->meta.framebuffer = v3dv_framebuffer_to_handle(state->framebuffer); |
| state->meta.pass = v3dv_render_pass_to_handle(state->pass); |
| |
| const uint32_t attachment_state_item_size = |
| sizeof(struct v3dv_cmd_buffer_attachment_state); |
| const uint32_t attachment_state_total_size = |
| attachment_state_item_size * state->attachment_alloc_count; |
| if (state->meta.attachment_alloc_count < state->attachment_alloc_count) { |
| if (state->meta.attachment_alloc_count > 0) |
| vk_free(&cmd_buffer->device->alloc, state->meta.attachments); |
| |
| state->meta.attachments = vk_zalloc(&cmd_buffer->device->alloc, |
| attachment_state_total_size, 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); |
| if (!state->meta.attachments) { |
| v3dv_flag_oom(cmd_buffer, NULL); |
| return; |
| } |
| state->meta.attachment_alloc_count = state->attachment_alloc_count; |
| } |
| state->meta.attachment_count = state->attachment_alloc_count; |
| memcpy(state->meta.attachments, state->attachments, |
| attachment_state_total_size); |
| |
| state->meta.tile_aligned_render_area = state->tile_aligned_render_area; |
| memcpy(&state->meta.render_area, &state->render_area, sizeof(VkRect2D)); |
| } |
| |
| state->meta.pipeline = v3dv_pipeline_to_handle(state->pipeline); |
| if (state->meta.pipeline) { |
| memcpy(&state->meta.dynamic, &state->dynamic, sizeof(state->dynamic)); |
| } |
| |
| /* We expect that meta operations are graphics-only and won't alter |
| * compute state. |
| */ |
| struct v3dv_descriptor_state *gfx_descriptor_state = |
| &state->descriptor_state[VK_PIPELINE_BIND_POINT_GRAPHICS]; |
| if (push_descriptor_state && gfx_descriptor_state->valid != 0) { |
| memcpy(&state->meta.descriptor_state, gfx_descriptor_state, |
| sizeof(state->descriptor_state)); |
| } |
| |
| /* FIXME: if we keep track of wether we have bound any push constant state |
| * at all we could restruct this only to cases where it is actually |
| * necessary. |
| */ |
| memcpy(state->meta.push_constants, cmd_buffer->push_constants_data, |
| sizeof(state->meta.push_constants)); |
| } |
| |
| /* This restores command buffer state after a meta operation |
| */ |
| void |
| v3dv_cmd_buffer_meta_state_pop(struct v3dv_cmd_buffer *cmd_buffer, |
| uint32_t dirty_dynamic_state) |
| { |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| |
| if (state->meta.subpass_idx != -1) { |
| state->pass = v3dv_render_pass_from_handle(state->meta.pass); |
| state->framebuffer = v3dv_framebuffer_from_handle(state->meta.framebuffer); |
| |
| assert(state->meta.attachment_count <= state->attachment_alloc_count); |
| const uint32_t attachment_state_item_size = |
| sizeof(struct v3dv_cmd_buffer_attachment_state); |
| const uint32_t attachment_state_total_size = |
| attachment_state_item_size * state->meta.attachment_count; |
| memcpy(state->attachments, state->meta.attachments, |
| attachment_state_total_size); |
| |
| state->tile_aligned_render_area = state->meta.tile_aligned_render_area; |
| memcpy(&state->render_area, &state->meta.render_area, sizeof(VkRect2D)); |
| |
| v3dv_cmd_buffer_subpass_resume(cmd_buffer, state->meta.subpass_idx); |
| } else { |
| state->subpass_idx = -1; |
| } |
| |
| if (state->meta.pipeline != VK_NULL_HANDLE) { |
| struct v3dv_pipeline *pipeline = |
| v3dv_pipeline_from_handle(state->meta.pipeline); |
| VkPipelineBindPoint pipeline_binding = |
| v3dv_pipeline_get_binding_point(pipeline); |
| v3dv_CmdBindPipeline(v3dv_cmd_buffer_to_handle(cmd_buffer), |
| pipeline_binding, |
| state->meta.pipeline); |
| if (pipeline_binding == VK_PIPELINE_BIND_POINT_GRAPHICS) { |
| memcpy(&state->dynamic, &state->meta.dynamic, sizeof(state->dynamic)); |
| state->dirty |= dirty_dynamic_state; |
| } |
| } else { |
| state->pipeline = VK_NULL_HANDLE; |
| } |
| |
| if (state->meta.descriptor_state.valid != 0) { |
| memcpy(&state->descriptor_state[VK_PIPELINE_BIND_POINT_GRAPHICS], |
| &state->meta.descriptor_state, |
| sizeof(state->descriptor_state)); |
| } else { |
| state->descriptor_state[VK_PIPELINE_BIND_POINT_GRAPHICS].valid = 0; |
| } |
| |
| memcpy(cmd_buffer->push_constants_data, state->meta.push_constants, |
| sizeof(state->meta.push_constants)); |
| |
| state->meta.pipeline = VK_NULL_HANDLE; |
| state->meta.framebuffer = VK_NULL_HANDLE; |
| state->meta.pass = VK_NULL_HANDLE; |
| state->meta.subpass_idx = -1; |
| state->meta.descriptor_state.valid = 0; |
| } |
| |
| /* FIXME: C&P from v3dx_draw. Refactor to common place? */ |
| static uint32_t |
| v3d_hw_prim_type(enum pipe_prim_type prim_type) |
| { |
| switch (prim_type) { |
| case PIPE_PRIM_POINTS: |
| case PIPE_PRIM_LINES: |
| case PIPE_PRIM_LINE_LOOP: |
| case PIPE_PRIM_LINE_STRIP: |
| case PIPE_PRIM_TRIANGLES: |
| case PIPE_PRIM_TRIANGLE_STRIP: |
| case PIPE_PRIM_TRIANGLE_FAN: |
| return prim_type; |
| |
| case PIPE_PRIM_LINES_ADJACENCY: |
| case PIPE_PRIM_LINE_STRIP_ADJACENCY: |
| case PIPE_PRIM_TRIANGLES_ADJACENCY: |
| case PIPE_PRIM_TRIANGLE_STRIP_ADJACENCY: |
| return 8 + (prim_type - PIPE_PRIM_LINES_ADJACENCY); |
| |
| default: |
| unreachable("Unsupported primitive type"); |
| } |
| } |
| |
| struct v3dv_draw_info { |
| uint32_t vertex_count; |
| uint32_t instance_count; |
| uint32_t first_vertex; |
| uint32_t first_instance; |
| }; |
| |
| static void |
| cmd_buffer_emit_draw(struct v3dv_cmd_buffer *cmd_buffer, |
| struct v3dv_draw_info *info) |
| { |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| struct v3dv_pipeline *pipeline = state->pipeline; |
| |
| assert(pipeline); |
| |
| uint32_t hw_prim_type = v3d_hw_prim_type(pipeline->vs->topology); |
| |
| if (info->first_instance > 0) { |
| v3dv_cl_ensure_space_with_branch( |
| &job->bcl, cl_packet_length(BASE_VERTEX_BASE_INSTANCE)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit(&job->bcl, BASE_VERTEX_BASE_INSTANCE, base) { |
| base.base_instance = info->first_instance; |
| base.base_vertex = 0; |
| } |
| } |
| |
| if (info->instance_count > 1) { |
| v3dv_cl_ensure_space_with_branch( |
| &job->bcl, cl_packet_length(VERTEX_ARRAY_INSTANCED_PRIMS)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit(&job->bcl, VERTEX_ARRAY_INSTANCED_PRIMS, prim) { |
| prim.mode = hw_prim_type; |
| prim.index_of_first_vertex = info->first_vertex; |
| prim.number_of_instances = info->instance_count; |
| prim.instance_length = info->vertex_count; |
| } |
| } else { |
| v3dv_cl_ensure_space_with_branch( |
| &job->bcl, cl_packet_length(VERTEX_ARRAY_PRIMS)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit(&job->bcl, VERTEX_ARRAY_PRIMS, prim) { |
| prim.mode = hw_prim_type; |
| prim.length = info->vertex_count; |
| prim.index_of_first_vertex = info->first_vertex; |
| } |
| } |
| } |
| |
| static struct v3dv_job * |
| cmd_buffer_pre_draw_split_job(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| /* If the job has been flagged with 'always_flush' and it has already |
| * recorded any draw calls then we need to start a new job for it. |
| */ |
| if (job->always_flush && job->draw_count > 0) { |
| assert(cmd_buffer->state.pass); |
| /* First, flag the current job as not being the last in the |
| * current subpass |
| */ |
| job->is_subpass_finish = false; |
| |
| /* Now start a new job in the same subpass and flag it as continuing |
| * the current subpass. |
| */ |
| job = v3dv_cmd_buffer_subpass_resume(cmd_buffer, |
| cmd_buffer->state.subpass_idx); |
| assert(job->draw_count == 0); |
| |
| /* Inherit the 'always flush' behavior */ |
| job->always_flush = true; |
| } |
| |
| assert(job->draw_count == 0 || !job->always_flush); |
| return job; |
| } |
| |
| static void |
| cmd_buffer_emit_pre_draw(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| assert(cmd_buffer->state.pipeline); |
| assert(!(cmd_buffer->state.pipeline->active_stages & VK_SHADER_STAGE_COMPUTE_BIT)); |
| |
| /* If the job is configured to flush on every draw call we need to create |
| * a new job now. |
| */ |
| struct v3dv_job *job = cmd_buffer_pre_draw_split_job(cmd_buffer); |
| job->draw_count++; |
| |
| /* We may need to compile shader variants based on bound textures */ |
| uint32_t *dirty = &cmd_buffer->state.dirty; |
| if (*dirty & (V3DV_CMD_DIRTY_PIPELINE | |
| V3DV_CMD_DIRTY_DESCRIPTOR_SETS)) { |
| update_pipeline_variants(cmd_buffer); |
| } |
| |
| /* GL shader state binds shaders, uniform and vertex attribute state. The |
| * compiler injects uniforms to handle some descriptor types (such as |
| * textures), so we need to regen that when descriptor state changes. |
| */ |
| if (*dirty & (V3DV_CMD_DIRTY_PIPELINE | |
| V3DV_CMD_DIRTY_VERTEX_BUFFER | |
| V3DV_CMD_DIRTY_DESCRIPTOR_SETS | |
| V3DV_CMD_DIRTY_PUSH_CONSTANTS)) { |
| emit_gl_shader_state(cmd_buffer); |
| } |
| |
| if (*dirty & (V3DV_CMD_DIRTY_PIPELINE)) { |
| emit_configuration_bits(cmd_buffer); |
| emit_varyings_state(cmd_buffer); |
| } |
| |
| if (*dirty & (V3DV_CMD_DIRTY_VIEWPORT | V3DV_CMD_DIRTY_SCISSOR)) { |
| emit_scissor(cmd_buffer); |
| } |
| |
| if (*dirty & V3DV_CMD_DIRTY_VIEWPORT) { |
| emit_viewport(cmd_buffer); |
| } |
| |
| const uint32_t dynamic_stencil_dirty_flags = |
| V3DV_CMD_DIRTY_STENCIL_COMPARE_MASK | |
| V3DV_CMD_DIRTY_STENCIL_WRITE_MASK | |
| V3DV_CMD_DIRTY_STENCIL_REFERENCE; |
| if (*dirty & (V3DV_CMD_DIRTY_PIPELINE | dynamic_stencil_dirty_flags)) |
| emit_stencil(cmd_buffer); |
| |
| if (*dirty & (V3DV_CMD_DIRTY_PIPELINE | V3DV_CMD_DIRTY_DEPTH_BIAS)) |
| emit_depth_bias(cmd_buffer); |
| |
| if (*dirty & (V3DV_CMD_DIRTY_PIPELINE | V3DV_CMD_DIRTY_BLEND_CONSTANTS)) |
| emit_blend(cmd_buffer); |
| |
| if (*dirty & V3DV_CMD_DIRTY_OCCLUSION_QUERY) |
| emit_occlusion_query(cmd_buffer); |
| |
| if (*dirty & V3DV_CMD_DIRTY_LINE_WIDTH) |
| emit_line_width(cmd_buffer); |
| |
| cmd_buffer->state.dirty &= ~V3DV_CMD_DIRTY_PIPELINE; |
| } |
| |
| static void |
| cmd_buffer_draw(struct v3dv_cmd_buffer *cmd_buffer, |
| struct v3dv_draw_info *info) |
| { |
| cmd_buffer_emit_pre_draw(cmd_buffer); |
| cmd_buffer_emit_draw(cmd_buffer, info); |
| } |
| |
| void |
| v3dv_CmdDraw(VkCommandBuffer commandBuffer, |
| uint32_t vertexCount, |
| uint32_t instanceCount, |
| uint32_t firstVertex, |
| uint32_t firstInstance) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| struct v3dv_draw_info info = {}; |
| info.vertex_count = vertexCount; |
| info.instance_count = instanceCount; |
| info.first_instance = firstInstance; |
| info.first_vertex = firstVertex; |
| |
| cmd_buffer_draw(cmd_buffer, &info); |
| } |
| |
| void |
| v3dv_CmdDrawIndexed(VkCommandBuffer commandBuffer, |
| uint32_t indexCount, |
| uint32_t instanceCount, |
| uint32_t firstIndex, |
| int32_t vertexOffset, |
| uint32_t firstInstance) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| |
| cmd_buffer_emit_pre_draw(cmd_buffer); |
| |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| const struct v3dv_pipeline *pipeline = cmd_buffer->state.pipeline; |
| uint32_t hw_prim_type = v3d_hw_prim_type(pipeline->vs->topology); |
| uint8_t index_type = ffs(cmd_buffer->state.index_buffer.index_size) - 1; |
| uint32_t index_offset = firstIndex * cmd_buffer->state.index_buffer.index_size; |
| |
| if (vertexOffset != 0 || firstInstance != 0) { |
| v3dv_cl_ensure_space_with_branch( |
| &job->bcl, cl_packet_length(BASE_VERTEX_BASE_INSTANCE)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit(&job->bcl, BASE_VERTEX_BASE_INSTANCE, base) { |
| base.base_instance = firstInstance; |
| base.base_vertex = vertexOffset; |
| } |
| } |
| |
| if (instanceCount == 1) { |
| v3dv_cl_ensure_space_with_branch( |
| &job->bcl, cl_packet_length(INDEXED_PRIM_LIST)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit(&job->bcl, INDEXED_PRIM_LIST, prim) { |
| prim.index_type = index_type; |
| prim.length = indexCount; |
| prim.index_offset = index_offset; |
| prim.mode = hw_prim_type; |
| prim.enable_primitive_restarts = pipeline->primitive_restart; |
| } |
| } else if (instanceCount > 1) { |
| v3dv_cl_ensure_space_with_branch( |
| &job->bcl, cl_packet_length(INDEXED_INSTANCED_PRIM_LIST)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit(&job->bcl, INDEXED_INSTANCED_PRIM_LIST, prim) { |
| prim.index_type = index_type; |
| prim.index_offset = index_offset; |
| prim.mode = hw_prim_type; |
| prim.enable_primitive_restarts = pipeline->primitive_restart; |
| prim.number_of_instances = instanceCount; |
| prim.instance_length = indexCount; |
| } |
| } |
| } |
| |
| void |
| v3dv_CmdDrawIndirect(VkCommandBuffer commandBuffer, |
| VkBuffer _buffer, |
| VkDeviceSize offset, |
| uint32_t drawCount, |
| uint32_t stride) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| V3DV_FROM_HANDLE(v3dv_buffer, buffer, _buffer); |
| |
| /* drawCount is the number of draws to execute, and can be zero. */ |
| if (drawCount == 0) |
| return; |
| |
| cmd_buffer_emit_pre_draw(cmd_buffer); |
| |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| const struct v3dv_pipeline *pipeline = cmd_buffer->state.pipeline; |
| uint32_t hw_prim_type = v3d_hw_prim_type(pipeline->vs->topology); |
| |
| v3dv_cl_ensure_space_with_branch( |
| &job->bcl, cl_packet_length(INDIRECT_VERTEX_ARRAY_INSTANCED_PRIMS)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit(&job->bcl, INDIRECT_VERTEX_ARRAY_INSTANCED_PRIMS, prim) { |
| prim.mode = hw_prim_type; |
| prim.number_of_draw_indirect_array_records = drawCount; |
| prim.stride_in_multiples_of_4_bytes = stride >> 2; |
| prim.address = v3dv_cl_address(buffer->mem->bo, |
| buffer->mem_offset + offset); |
| } |
| } |
| |
| void |
| v3dv_CmdDrawIndexedIndirect(VkCommandBuffer commandBuffer, |
| VkBuffer _buffer, |
| VkDeviceSize offset, |
| uint32_t drawCount, |
| uint32_t stride) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| V3DV_FROM_HANDLE(v3dv_buffer, buffer, _buffer); |
| |
| /* drawCount is the number of draws to execute, and can be zero. */ |
| if (drawCount == 0) |
| return; |
| |
| cmd_buffer_emit_pre_draw(cmd_buffer); |
| |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| const struct v3dv_pipeline *pipeline = cmd_buffer->state.pipeline; |
| uint32_t hw_prim_type = v3d_hw_prim_type(pipeline->vs->topology); |
| uint8_t index_type = ffs(cmd_buffer->state.index_buffer.index_size) - 1; |
| |
| v3dv_cl_ensure_space_with_branch( |
| &job->bcl, cl_packet_length(INDIRECT_INDEXED_INSTANCED_PRIM_LIST)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| cl_emit(&job->bcl, INDIRECT_INDEXED_INSTANCED_PRIM_LIST, prim) { |
| prim.index_type = index_type; |
| prim.mode = hw_prim_type; |
| prim.enable_primitive_restarts = pipeline->primitive_restart; |
| prim.number_of_draw_indirect_indexed_records = drawCount; |
| prim.stride_in_multiples_of_4_bytes = stride >> 2; |
| prim.address = v3dv_cl_address(buffer->mem->bo, |
| buffer->mem_offset + offset); |
| } |
| } |
| |
| void |
| v3dv_CmdPipelineBarrier(VkCommandBuffer commandBuffer, |
| VkPipelineStageFlags srcStageMask, |
| VkPipelineStageFlags dstStageMask, |
| VkDependencyFlags dependencyFlags, |
| uint32_t memoryBarrierCount, |
| const VkMemoryBarrier *pMemoryBarriers, |
| uint32_t bufferMemoryBarrierCount, |
| const VkBufferMemoryBarrier *pBufferMemoryBarriers, |
| uint32_t imageMemoryBarrierCount, |
| const VkImageMemoryBarrier *pImageMemoryBarriers) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| |
| struct v3dv_job *job = cmd_buffer->state.job; |
| if (!job) |
| return; |
| |
| v3dv_cmd_buffer_finish_job(cmd_buffer); |
| } |
| |
| void |
| v3dv_CmdBindVertexBuffers(VkCommandBuffer commandBuffer, |
| uint32_t firstBinding, |
| uint32_t bindingCount, |
| const VkBuffer *pBuffers, |
| const VkDeviceSize *pOffsets) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| struct v3dv_vertex_binding *vb = cmd_buffer->state.vertex_bindings; |
| |
| /* We have to defer setting up vertex buffer since we need the buffer |
| * stride from the pipeline. |
| */ |
| |
| assert(firstBinding + bindingCount <= MAX_VBS); |
| bool vb_state_changed = false; |
| for (uint32_t i = 0; i < bindingCount; i++) { |
| if (vb[firstBinding + i].buffer != v3dv_buffer_from_handle(pBuffers[i])) { |
| vb[firstBinding + i].buffer = v3dv_buffer_from_handle(pBuffers[i]); |
| vb_state_changed = true; |
| } |
| if (vb[firstBinding + i].offset != pOffsets[i]) { |
| vb[firstBinding + i].offset = pOffsets[i]; |
| vb_state_changed = true; |
| } |
| } |
| |
| if (vb_state_changed) |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_VERTEX_BUFFER; |
| } |
| |
| static uint32_t |
| get_index_size(VkIndexType index_type) |
| { |
| switch (index_type) { |
| case VK_INDEX_TYPE_UINT16: |
| return 2; |
| break; |
| case VK_INDEX_TYPE_UINT32: |
| return 4; |
| break; |
| default: |
| unreachable("Unsupported index type"); |
| } |
| } |
| |
| void |
| v3dv_CmdBindIndexBuffer(VkCommandBuffer commandBuffer, |
| VkBuffer buffer, |
| VkDeviceSize offset, |
| VkIndexType indexType) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| V3DV_FROM_HANDLE(v3dv_buffer, ibuffer, buffer); |
| |
| struct v3dv_job *job = cmd_buffer->state.job; |
| assert(job); |
| |
| v3dv_cl_ensure_space_with_branch( |
| &job->bcl, cl_packet_length(INDEX_BUFFER_SETUP)); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| const uint32_t index_size = get_index_size(indexType); |
| |
| /* If we have started a new job we always need to emit index buffer state. |
| * We know we are in that scenario because that is the only case where we |
| * set the dirty bit. |
| */ |
| if (!(cmd_buffer->state.dirty & V3DV_CMD_DIRTY_INDEX_BUFFER)) { |
| if (buffer == cmd_buffer->state.index_buffer.buffer && |
| offset == cmd_buffer->state.index_buffer.offset && |
| index_size == cmd_buffer->state.index_buffer.index_size) { |
| return; |
| } |
| } |
| |
| cl_emit(&job->bcl, INDEX_BUFFER_SETUP, ib) { |
| ib.address = v3dv_cl_address(ibuffer->mem->bo, |
| ibuffer->mem_offset + offset); |
| ib.size = ibuffer->mem->bo->size; |
| } |
| |
| cmd_buffer->state.index_buffer.buffer = buffer; |
| cmd_buffer->state.index_buffer.offset = offset; |
| cmd_buffer->state.index_buffer.index_size = index_size; |
| |
| cmd_buffer->state.dirty &= ~V3DV_CMD_DIRTY_INDEX_BUFFER; |
| } |
| |
| void |
| v3dv_CmdSetStencilCompareMask(VkCommandBuffer commandBuffer, |
| VkStencilFaceFlags faceMask, |
| uint32_t compareMask) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| |
| if (faceMask & VK_STENCIL_FACE_FRONT_BIT) |
| cmd_buffer->state.dynamic.stencil_compare_mask.front = compareMask & 0xff; |
| if (faceMask & VK_STENCIL_FACE_BACK_BIT) |
| cmd_buffer->state.dynamic.stencil_compare_mask.back = compareMask & 0xff; |
| |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_STENCIL_COMPARE_MASK; |
| } |
| |
| void |
| v3dv_CmdSetStencilWriteMask(VkCommandBuffer commandBuffer, |
| VkStencilFaceFlags faceMask, |
| uint32_t writeMask) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| |
| if (faceMask & VK_STENCIL_FACE_FRONT_BIT) |
| cmd_buffer->state.dynamic.stencil_write_mask.front = writeMask & 0xff; |
| if (faceMask & VK_STENCIL_FACE_BACK_BIT) |
| cmd_buffer->state.dynamic.stencil_write_mask.back = writeMask & 0xff; |
| |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_STENCIL_WRITE_MASK; |
| } |
| |
| void |
| v3dv_CmdSetStencilReference(VkCommandBuffer commandBuffer, |
| VkStencilFaceFlags faceMask, |
| uint32_t reference) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| |
| if (faceMask & VK_STENCIL_FACE_FRONT_BIT) |
| cmd_buffer->state.dynamic.stencil_reference.front = reference & 0xff; |
| if (faceMask & VK_STENCIL_FACE_BACK_BIT) |
| cmd_buffer->state.dynamic.stencil_reference.back = reference & 0xff; |
| |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_STENCIL_REFERENCE; |
| } |
| |
| void |
| v3dv_CmdSetDepthBias(VkCommandBuffer commandBuffer, |
| float depthBiasConstantFactor, |
| float depthBiasClamp, |
| float depthBiasSlopeFactor) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| |
| cmd_buffer->state.dynamic.depth_bias.constant_factor = depthBiasConstantFactor; |
| cmd_buffer->state.dynamic.depth_bias.slope_factor = depthBiasSlopeFactor; |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_DEPTH_BIAS; |
| } |
| |
| void |
| v3dv_CmdSetDepthBounds(VkCommandBuffer commandBuffer, |
| float minDepthBounds, |
| float maxDepthBounds) |
| { |
| /* We do not support depth bounds testing so we just ingore this. We are |
| * already asserting that pipelines don't enable the feature anyway. |
| */ |
| } |
| |
| void |
| v3dv_CmdSetLineWidth(VkCommandBuffer commandBuffer, |
| float lineWidth) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| |
| cmd_buffer->state.dynamic.line_width = lineWidth; |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_LINE_WIDTH; |
| } |
| |
| void |
| v3dv_CmdBindDescriptorSets(VkCommandBuffer commandBuffer, |
| VkPipelineBindPoint pipelineBindPoint, |
| VkPipelineLayout _layout, |
| uint32_t firstSet, |
| uint32_t descriptorSetCount, |
| const VkDescriptorSet *pDescriptorSets, |
| uint32_t dynamicOffsetCount, |
| const uint32_t *pDynamicOffsets) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| V3DV_FROM_HANDLE(v3dv_pipeline_layout, layout, _layout); |
| |
| uint32_t dyn_index = 0; |
| |
| assert(firstSet + descriptorSetCount <= MAX_SETS); |
| |
| struct v3dv_descriptor_state *descriptor_state = |
| &cmd_buffer->state.descriptor_state[pipelineBindPoint]; |
| |
| bool descriptor_state_changed = false; |
| for (uint32_t i = 0; i < descriptorSetCount; i++) { |
| V3DV_FROM_HANDLE(v3dv_descriptor_set, set, pDescriptorSets[i]); |
| uint32_t index = firstSet + i; |
| |
| if (descriptor_state->descriptor_sets[index] != set) { |
| descriptor_state->descriptor_sets[index] = set; |
| descriptor_state_changed = true; |
| } |
| |
| if (!(descriptor_state->valid & (1u << index))) { |
| descriptor_state->valid |= (1u << index); |
| descriptor_state_changed = true; |
| } |
| |
| for (uint32_t j = 0; j < set->layout->dynamic_offset_count; j++, dyn_index++) { |
| uint32_t idx = j + layout->set[i + firstSet].dynamic_offset_start; |
| |
| if (descriptor_state->dynamic_offsets[idx] != pDynamicOffsets[dyn_index]) { |
| descriptor_state->dynamic_offsets[idx] = pDynamicOffsets[dyn_index]; |
| descriptor_state_changed = true; |
| } |
| } |
| } |
| |
| if (descriptor_state_changed) { |
| if (pipelineBindPoint == VK_PIPELINE_BIND_POINT_GRAPHICS) |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_DESCRIPTOR_SETS; |
| else |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_COMPUTE_DESCRIPTOR_SETS; |
| } |
| } |
| |
| void |
| v3dv_CmdPushConstants(VkCommandBuffer commandBuffer, |
| VkPipelineLayout layout, |
| VkShaderStageFlags stageFlags, |
| uint32_t offset, |
| uint32_t size, |
| const void *pValues) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| |
| if (!memcmp(cmd_buffer->push_constants_data + offset, pValues, size)) |
| return; |
| |
| memcpy((void*) cmd_buffer->push_constants_data + offset, pValues, size); |
| |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_PUSH_CONSTANTS; |
| } |
| |
| void |
| v3dv_CmdSetBlendConstants(VkCommandBuffer commandBuffer, |
| const float blendConstants[4]) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| |
| if (!memcmp(state->dynamic.blend_constants, blendConstants, |
| sizeof(state->dynamic.blend_constants))) { |
| return; |
| } |
| |
| memcpy(state->dynamic.blend_constants, blendConstants, |
| sizeof(state->dynamic.blend_constants)); |
| |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_BLEND_CONSTANTS; |
| } |
| |
| void |
| v3dv_cmd_buffer_reset_queries(struct v3dv_cmd_buffer *cmd_buffer, |
| struct v3dv_query_pool *pool, |
| uint32_t first, |
| uint32_t count) |
| { |
| /* Resets can only happen outside a render pass instance so we should not |
| * be in the middle of job recording. |
| */ |
| assert(cmd_buffer->state.pass == NULL); |
| assert(cmd_buffer->state.job == NULL); |
| |
| assert(first < pool->query_count); |
| assert(first + count <= pool->query_count); |
| |
| struct v3dv_job *job = |
| v3dv_cmd_buffer_create_cpu_job(cmd_buffer->device, |
| V3DV_JOB_TYPE_CPU_RESET_QUERIES, |
| cmd_buffer, -1); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| job->cpu.query_reset.pool = pool; |
| job->cpu.query_reset.first = first; |
| job->cpu.query_reset.count = count; |
| |
| list_addtail(&job->list_link, &cmd_buffer->jobs); |
| } |
| |
| static void |
| ensure_array_state(struct v3dv_cmd_buffer *cmd_buffer, |
| uint32_t slot_size, |
| uint32_t used_count, |
| uint32_t *alloc_count, |
| void **ptr) |
| { |
| if (used_count >= *alloc_count) { |
| const uint32_t prev_slot_count = *alloc_count; |
| void *old_buffer = *ptr; |
| |
| const uint32_t new_slot_count = MAX2(*alloc_count * 2, 4); |
| const uint32_t bytes = new_slot_count * slot_size; |
| *ptr = vk_alloc(&cmd_buffer->device->alloc, bytes, 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); |
| if (*ptr == NULL) { |
| fprintf(stderr, "Error: failed to allocate CPU buffer for query.\n"); |
| v3dv_flag_oom(cmd_buffer, NULL); |
| return; |
| } |
| |
| memcpy(*ptr, old_buffer, prev_slot_count * slot_size); |
| *alloc_count = new_slot_count; |
| } |
| assert(used_count < *alloc_count); |
| } |
| |
| void |
| v3dv_cmd_buffer_begin_query(struct v3dv_cmd_buffer *cmd_buffer, |
| struct v3dv_query_pool *pool, |
| uint32_t query, |
| VkQueryControlFlags flags) |
| { |
| /* FIXME: we only support one active query for now */ |
| assert(cmd_buffer->state.query.active_query == NULL); |
| assert(query < pool->query_count); |
| |
| cmd_buffer->state.query.active_query = pool->queries[query].bo; |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_OCCLUSION_QUERY; |
| } |
| |
| void |
| v3dv_cmd_buffer_end_query(struct v3dv_cmd_buffer *cmd_buffer, |
| struct v3dv_query_pool *pool, |
| uint32_t query) |
| { |
| assert(query < pool->query_count); |
| assert(cmd_buffer->state.query.active_query != NULL); |
| |
| if (cmd_buffer->state.pass) { |
| /* Queue the EndQuery in the command buffer state, we will create a CPU |
| * job to flag all of these queries as possibly available right after the |
| * render pass job in which they have been recorded. |
| */ |
| struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; |
| ensure_array_state(cmd_buffer, |
| sizeof(struct v3dv_end_query_cpu_job_info), |
| state->query.end.used_count, |
| &state->query.end.alloc_count, |
| (void **) &state->query.end.states); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| struct v3dv_end_query_cpu_job_info *info = |
| &state->query.end.states[state->query.end.used_count++]; |
| |
| info->pool = pool; |
| info->query = query; |
| } else { |
| /* Otherwise, schedule the CPU job immediately */ |
| struct v3dv_job *job = |
| v3dv_cmd_buffer_create_cpu_job(cmd_buffer->device, |
| V3DV_JOB_TYPE_CPU_END_QUERY, |
| cmd_buffer, -1); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| job->cpu.query_end.pool = pool; |
| job->cpu.query_end.query = query; |
| list_addtail(&job->list_link, &cmd_buffer->jobs); |
| } |
| |
| cmd_buffer->state.query.active_query = NULL; |
| cmd_buffer->state.dirty |= V3DV_CMD_DIRTY_OCCLUSION_QUERY; |
| } |
| |
| void |
| v3dv_cmd_buffer_copy_query_results(struct v3dv_cmd_buffer *cmd_buffer, |
| struct v3dv_query_pool *pool, |
| uint32_t first, |
| uint32_t count, |
| struct v3dv_buffer *dst, |
| uint32_t offset, |
| uint32_t stride, |
| VkQueryResultFlags flags) |
| { |
| /* Copies can only happen outside a render pass instance so we should not |
| * be in the middle of job recording. |
| */ |
| assert(cmd_buffer->state.pass == NULL); |
| assert(cmd_buffer->state.job == NULL); |
| |
| assert(first < pool->query_count); |
| assert(first + count <= pool->query_count); |
| |
| struct v3dv_job *job = |
| v3dv_cmd_buffer_create_cpu_job(cmd_buffer->device, |
| V3DV_JOB_TYPE_CPU_COPY_QUERY_RESULTS, |
| cmd_buffer, -1); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| job->cpu.query_copy_results.pool = pool; |
| job->cpu.query_copy_results.first = first; |
| job->cpu.query_copy_results.count = count; |
| job->cpu.query_copy_results.dst = dst; |
| job->cpu.query_copy_results.offset = offset; |
| job->cpu.query_copy_results.stride = stride; |
| job->cpu.query_copy_results.flags = flags; |
| |
| list_addtail(&job->list_link, &cmd_buffer->jobs); |
| } |
| |
| void |
| v3dv_cmd_buffer_add_tfu_job(struct v3dv_cmd_buffer *cmd_buffer, |
| struct drm_v3d_submit_tfu *tfu) |
| { |
| struct v3dv_device *device = cmd_buffer->device; |
| struct v3dv_job *job = vk_zalloc(&device->alloc, |
| sizeof(struct v3dv_job), 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); |
| if (!job) { |
| v3dv_flag_oom(cmd_buffer, NULL); |
| return; |
| } |
| |
| v3dv_job_init(job, V3DV_JOB_TYPE_GPU_TFU, device, cmd_buffer, -1); |
| job->tfu = *tfu; |
| list_addtail(&job->list_link, &cmd_buffer->jobs); |
| } |
| |
| void |
| v3dv_CmdSetEvent(VkCommandBuffer commandBuffer, |
| VkEvent _event, |
| VkPipelineStageFlags stageMask) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| V3DV_FROM_HANDLE(v3dv_event, event, _event); |
| |
| /* Event (re)sets can only happen outside a render pass instance so we |
| * should not be in the middle of job recording. |
| */ |
| assert(cmd_buffer->state.pass == NULL); |
| assert(cmd_buffer->state.job == NULL); |
| |
| struct v3dv_job *job = |
| v3dv_cmd_buffer_create_cpu_job(cmd_buffer->device, |
| V3DV_JOB_TYPE_CPU_SET_EVENT, |
| cmd_buffer, -1); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| job->cpu.event_set.event = event; |
| job->cpu.event_set.state = 1; |
| |
| list_addtail(&job->list_link, &cmd_buffer->jobs); |
| } |
| |
| void |
| v3dv_CmdResetEvent(VkCommandBuffer commandBuffer, |
| VkEvent _event, |
| VkPipelineStageFlags stageMask) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| V3DV_FROM_HANDLE(v3dv_event, event, _event); |
| |
| /* Event (re)sets can only happen outside a render pass instance so we |
| * should not be in the middle of job recording. |
| */ |
| assert(cmd_buffer->state.pass == NULL); |
| assert(cmd_buffer->state.job == NULL); |
| |
| struct v3dv_job *job = |
| v3dv_cmd_buffer_create_cpu_job(cmd_buffer->device, |
| V3DV_JOB_TYPE_CPU_SET_EVENT, |
| cmd_buffer, -1); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| job->cpu.event_set.event = event; |
| job->cpu.event_set.state = 0; |
| |
| list_addtail(&job->list_link, &cmd_buffer->jobs); |
| } |
| |
| void |
| v3dv_CmdWaitEvents(VkCommandBuffer commandBuffer, |
| uint32_t eventCount, |
| const VkEvent *pEvents, |
| VkPipelineStageFlags srcStageMask, |
| VkPipelineStageFlags dstStageMask, |
| uint32_t memoryBarrierCount, |
| const VkMemoryBarrier *pMemoryBarriers, |
| uint32_t bufferMemoryBarrierCount, |
| const VkBufferMemoryBarrier *pBufferMemoryBarriers, |
| uint32_t imageMemoryBarrierCount, |
| const VkImageMemoryBarrier *pImageMemoryBarriers) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| |
| assert(eventCount > 0); |
| |
| struct v3dv_job *job = |
| v3dv_cmd_buffer_create_cpu_job(cmd_buffer->device, |
| V3DV_JOB_TYPE_CPU_WAIT_EVENTS, |
| cmd_buffer, -1); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| const uint32_t event_list_size = sizeof(struct v3dv_event *) * eventCount; |
| |
| job->cpu.event_wait.events = |
| vk_alloc(&cmd_buffer->device->alloc, event_list_size, 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); |
| if (!job->cpu.event_wait.events) { |
| v3dv_flag_oom(cmd_buffer, NULL); |
| return; |
| } |
| job->cpu.event_wait.event_count = eventCount; |
| |
| for (uint32_t i = 0; i < eventCount; i++) |
| job->cpu.event_wait.events[i] = v3dv_event_from_handle(pEvents[i]); |
| |
| /* vkCmdWaitEvents can be recorded inside a render pass, so we might have |
| * an active job. |
| * |
| * If we are inside a render pass, because we vkCmd(Re)SetEvent can't happen |
| * inside a render pass, it is safe to move the wait job so it happens right |
| * before the current job we are currently recording for the subpass, if any |
| * (it would actually be safe to move it all the way back to right before |
| * the start of the render pass). |
| * |
| * If we are outside a render pass then we should not have any on-going job |
| * and we are free to just add the wait job without restrictions. |
| */ |
| assert(cmd_buffer->state.pass || !cmd_buffer->state.job); |
| list_addtail(&job->list_link, &cmd_buffer->jobs); |
| } |
| |
| void |
| v3dv_CmdWriteTimestamp(VkCommandBuffer commandBuffer, |
| VkPipelineStageFlagBits pipelineStage, |
| VkQueryPool queryPool, |
| uint32_t query) |
| { |
| unreachable("Timestamp queries are not supported."); |
| } |
| |
| static void |
| cmd_buffer_emit_pre_dispatch(struct v3dv_cmd_buffer *cmd_buffer) |
| { |
| assert(cmd_buffer->state.pipeline); |
| assert(cmd_buffer->state.pipeline->active_stages == VK_SHADER_STAGE_COMPUTE_BIT); |
| |
| /* We may need to compile shader variants based on bound textures */ |
| uint32_t *dirty = &cmd_buffer->state.dirty; |
| if (*dirty & (V3DV_CMD_DIRTY_PIPELINE | |
| V3DV_CMD_DIRTY_COMPUTE_DESCRIPTOR_SETS)) { |
| update_pipeline_variants(cmd_buffer); |
| } |
| |
| *dirty &= ~(V3DV_CMD_DIRTY_PIPELINE | |
| V3DV_CMD_DIRTY_COMPUTE_DESCRIPTOR_SETS); |
| } |
| |
| #define V3D_CSD_CFG012_WG_COUNT_SHIFT 16 |
| #define V3D_CSD_CFG012_WG_OFFSET_SHIFT 0 |
| /* Allow this dispatch to start while the last one is still running. */ |
| #define V3D_CSD_CFG3_OVERLAP_WITH_PREV (1 << 26) |
| /* Maximum supergroup ID. 6 bits. */ |
| #define V3D_CSD_CFG3_MAX_SG_ID_SHIFT 20 |
| /* Batches per supergroup minus 1. 8 bits. */ |
| #define V3D_CSD_CFG3_BATCHES_PER_SG_M1_SHIFT 12 |
| /* Workgroups per supergroup, 0 means 16 */ |
| #define V3D_CSD_CFG3_WGS_PER_SG_SHIFT 8 |
| #define V3D_CSD_CFG3_WG_SIZE_SHIFT 0 |
| |
| #define V3D_CSD_CFG5_PROPAGATE_NANS (1 << 2) |
| #define V3D_CSD_CFG5_SINGLE_SEG (1 << 1) |
| #define V3D_CSD_CFG5_THREADING (1 << 0) |
| |
| void |
| v3dv_cmd_buffer_rewrite_indirect_csd_job( |
| struct v3dv_csd_indirect_cpu_job_info *info, |
| const uint32_t *wg_counts) |
| { |
| assert(info->csd_job); |
| struct v3dv_job *job = info->csd_job; |
| |
| assert(job->type == V3DV_JOB_TYPE_GPU_CSD); |
| assert(wg_counts[0] > 0 && wg_counts[1] > 0 && wg_counts[2] > 0); |
| |
| struct drm_v3d_submit_csd *submit = &job->csd.submit; |
| |
| job->csd.wg_count[0] = wg_counts[0]; |
| job->csd.wg_count[1] = wg_counts[1]; |
| job->csd.wg_count[2] = wg_counts[2]; |
| |
| submit->cfg[0] = wg_counts[0] << V3D_CSD_CFG012_WG_COUNT_SHIFT; |
| submit->cfg[1] = wg_counts[1] << V3D_CSD_CFG012_WG_COUNT_SHIFT; |
| submit->cfg[2] = wg_counts[2] << V3D_CSD_CFG012_WG_COUNT_SHIFT; |
| |
| submit->cfg[4] = DIV_ROUND_UP(info->wg_size, 16) * |
| (wg_counts[0] * wg_counts[1] * wg_counts[2]) - 1; |
| assert(submit->cfg[4] != ~0); |
| |
| if (info->needs_wg_uniform_rewrite) { |
| /* Make sure the GPU is not currently accessing the indirect CL for this |
| * job, since we are about to overwrite some of the uniform data. |
| */ |
| const uint64_t infinite = 0xffffffffffffffffull; |
| v3dv_bo_wait(job->device, job->indirect.bo, infinite); |
| |
| for (uint32_t i = 0; i < 3; i++) { |
| if (info->wg_uniform_offsets[i]) { |
| /* Sanity check that our uniform pointers are within the allocated |
| * BO space for our indirect CL. |
| */ |
| assert(info->wg_uniform_offsets[i] >= (uint32_t *) job->indirect.base); |
| assert(info->wg_uniform_offsets[i] < (uint32_t *) job->indirect.next); |
| *(info->wg_uniform_offsets[i]) = wg_counts[i]; |
| } |
| } |
| } |
| } |
| |
| static struct v3dv_job * |
| cmd_buffer_create_csd_job(struct v3dv_cmd_buffer *cmd_buffer, |
| uint32_t group_count_x, |
| uint32_t group_count_y, |
| uint32_t group_count_z, |
| uint32_t **wg_uniform_offsets_out, |
| uint32_t *wg_size_out) |
| { |
| struct v3dv_pipeline *pipeline = cmd_buffer->state.pipeline; |
| assert(pipeline && pipeline->cs && pipeline->cs->nir); |
| |
| struct v3dv_job *job = vk_zalloc(&cmd_buffer->device->alloc, |
| sizeof(struct v3dv_job), 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); |
| if (!job) { |
| v3dv_flag_oom(cmd_buffer, NULL); |
| return NULL; |
| } |
| |
| v3dv_job_init(job, V3DV_JOB_TYPE_GPU_CSD, cmd_buffer->device, cmd_buffer, -1); |
| cmd_buffer->state.job = job; |
| |
| struct drm_v3d_submit_csd *submit = &job->csd.submit; |
| |
| job->csd.wg_count[0] = group_count_x; |
| job->csd.wg_count[1] = group_count_y; |
| job->csd.wg_count[2] = group_count_z; |
| |
| submit->cfg[0] |= group_count_x << V3D_CSD_CFG012_WG_COUNT_SHIFT; |
| submit->cfg[1] |= group_count_y << V3D_CSD_CFG012_WG_COUNT_SHIFT; |
| submit->cfg[2] |= group_count_z << V3D_CSD_CFG012_WG_COUNT_SHIFT; |
| |
| const struct nir_shader *cs = pipeline->cs->nir; |
| |
| const uint32_t wgs_per_sg = 1; /* FIXME */ |
| const uint32_t wg_size = cs->info.cs.local_size[0] * |
| cs->info.cs.local_size[1] * |
| cs->info.cs.local_size[2]; |
| submit->cfg[3] |= wgs_per_sg << V3D_CSD_CFG3_WGS_PER_SG_SHIFT; |
| submit->cfg[3] |= ((DIV_ROUND_UP(wgs_per_sg * wg_size, 16) - 1) << |
| V3D_CSD_CFG3_BATCHES_PER_SG_M1_SHIFT); |
| submit->cfg[3] |= (wg_size & 0xff) << V3D_CSD_CFG3_WG_SIZE_SHIFT; |
| if (wg_size_out) |
| *wg_size_out = wg_size; |
| |
| uint32_t batches_per_wg = DIV_ROUND_UP(wg_size, 16); |
| submit->cfg[4] = batches_per_wg * |
| (group_count_x * group_count_y * group_count_z) - 1; |
| assert(submit->cfg[4] != ~0); |
| |
| assert(pipeline->cs->current_variant && |
| pipeline->cs->current_variant->assembly_bo); |
| const struct v3dv_shader_variant *variant = pipeline->cs->current_variant; |
| submit->cfg[5] = variant->assembly_bo->offset; |
| submit->cfg[5] |= V3D_CSD_CFG5_PROPAGATE_NANS; |
| if (variant->prog_data.base->single_seg) |
| submit->cfg[5] |= V3D_CSD_CFG5_SINGLE_SEG; |
| if (variant->prog_data.base->threads == 4) |
| submit->cfg[5] |= V3D_CSD_CFG5_THREADING; |
| |
| if (variant->prog_data.cs->shared_size > 0) { |
| job->csd.shared_memory = |
| v3dv_bo_alloc(cmd_buffer->device, |
| variant->prog_data.cs->shared_size * wgs_per_sg, |
| "shared_vars", true); |
| if (!job->csd.shared_memory) { |
| v3dv_flag_oom(cmd_buffer, NULL); |
| return job; |
| } |
| } |
| |
| v3dv_job_add_bo(job, variant->assembly_bo); |
| |
| struct v3dv_cl_reloc uniforms = |
| v3dv_write_uniforms_wg_offsets(cmd_buffer, pipeline->cs, |
| wg_uniform_offsets_out); |
| submit->cfg[6] = uniforms.bo->offset + uniforms.offset; |
| |
| v3dv_job_add_bo(job, uniforms.bo); |
| |
| return job; |
| } |
| |
| static void |
| cmd_buffer_dispatch(struct v3dv_cmd_buffer *cmd_buffer, |
| uint32_t group_count_x, |
| uint32_t group_count_y, |
| uint32_t group_count_z) |
| { |
| if (group_count_x == 0 || group_count_y == 0 || group_count_z == 0) |
| return; |
| |
| struct v3dv_job *job = |
| cmd_buffer_create_csd_job(cmd_buffer, |
| group_count_x, |
| group_count_y, |
| group_count_z, |
| NULL, NULL); |
| |
| list_addtail(&job->list_link, &cmd_buffer->jobs); |
| cmd_buffer->state.job = NULL; |
| } |
| |
| void |
| v3dv_CmdDispatch(VkCommandBuffer commandBuffer, |
| uint32_t groupCountX, |
| uint32_t groupCountY, |
| uint32_t groupCountZ) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| |
| cmd_buffer_emit_pre_dispatch(cmd_buffer); |
| cmd_buffer_dispatch(cmd_buffer, groupCountX, groupCountY, groupCountZ); |
| } |
| |
| static void |
| cmd_buffer_dispatch_indirect(struct v3dv_cmd_buffer *cmd_buffer, |
| struct v3dv_buffer *buffer, |
| uint32_t offset) |
| { |
| /* We can't do indirect dispatches, so instead we record a CPU job that, |
| * when executed in the queue, will map the indirect buffer, read the |
| * dispatch parameters, and submit a regular dispatch. |
| */ |
| struct v3dv_job *job = |
| v3dv_cmd_buffer_create_cpu_job(cmd_buffer->device, |
| V3DV_JOB_TYPE_CPU_CSD_INDIRECT, |
| cmd_buffer, -1); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| |
| /* We need to create a CSD job now, even if we still don't know the actual |
| * dispatch parameters, because the job setup needs to be done using the |
| * current command buffer state (i.e. pipeline, descriptor sets, push |
| * constants, etc.). So we create the job with default dispatch parameters |
| * and we will rewrite the parts we need at submit time if the indirect |
| * parameters don't match the ones we used to setup the job. |
| */ |
| struct v3dv_job *csd_job = |
| cmd_buffer_create_csd_job(cmd_buffer, |
| 1, 1, 1, |
| &job->cpu.csd_indirect.wg_uniform_offsets[0], |
| &job->cpu.csd_indirect.wg_size); |
| v3dv_return_if_oom(cmd_buffer, NULL); |
| assert(csd_job); |
| |
| job->cpu.csd_indirect.buffer = buffer; |
| job->cpu.csd_indirect.offset = offset; |
| job->cpu.csd_indirect.csd_job = csd_job; |
| |
| /* If the compute shader reads the workgroup sizes we will also need to |
| * rewrite the corresponding uniforms. |
| */ |
| job->cpu.csd_indirect.needs_wg_uniform_rewrite = |
| job->cpu.csd_indirect.wg_uniform_offsets[0] || |
| job->cpu.csd_indirect.wg_uniform_offsets[1] || |
| job->cpu.csd_indirect.wg_uniform_offsets[2]; |
| |
| list_addtail(&job->list_link, &cmd_buffer->jobs); |
| cmd_buffer->state.job = NULL; |
| } |
| |
| void |
| v3dv_CmdDispatchIndirect(VkCommandBuffer commandBuffer, |
| VkBuffer _buffer, |
| VkDeviceSize offset) |
| { |
| V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer); |
| V3DV_FROM_HANDLE(v3dv_buffer, buffer, _buffer); |
| |
| assert(offset <= UINT32_MAX); |
| |
| cmd_buffer_emit_pre_dispatch(cmd_buffer); |
| cmd_buffer_dispatch_indirect(cmd_buffer, buffer, offset); |
| } |
| |
| void |
| v3dv_CmdResolveImage(VkCommandBuffer commandBuffer, |
| VkImage srcImage, |
| VkImageLayout srcImageLayout, |
| VkImage dstImage, |
| VkImageLayout dstImageLayout, |
| uint32_t regionCount, |
| const VkImageResolve *pRegions) |
| { |
| unreachable("vkCmdResolveImage not implemented"); |
| } |