| /* |
| * Copyright © 2015 Intel Corporation |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice (including the next |
| * paragraph) shall be included in all copies or substantial portions of the |
| * Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
| * IN THE SOFTWARE. |
| */ |
| |
| /** |
| * This file implements VkQueue, VkFence, and VkSemaphore |
| */ |
| |
| #include <errno.h> |
| #include <fcntl.h> |
| #include <unistd.h> |
| |
| #include "anv_private.h" |
| #include "vk_util.h" |
| |
| #include "genxml/gen7_pack.h" |
| |
| uint64_t anv_gettime_ns(void) |
| { |
| struct timespec current; |
| clock_gettime(CLOCK_MONOTONIC, ¤t); |
| return (uint64_t)current.tv_sec * NSEC_PER_SEC + current.tv_nsec; |
| } |
| |
| uint64_t anv_get_absolute_timeout(uint64_t timeout) |
| { |
| if (timeout == 0) |
| return 0; |
| uint64_t current_time = anv_gettime_ns(); |
| uint64_t max_timeout = (uint64_t) INT64_MAX - current_time; |
| |
| timeout = MIN2(max_timeout, timeout); |
| |
| return (current_time + timeout); |
| } |
| |
| static int64_t anv_get_relative_timeout(uint64_t abs_timeout) |
| { |
| uint64_t now = anv_gettime_ns(); |
| |
| /* We don't want negative timeouts. |
| * |
| * DRM_IOCTL_I915_GEM_WAIT uses a signed 64 bit timeout and is |
| * supposed to block indefinitely timeouts < 0. Unfortunately, |
| * this was broken for a couple of kernel releases. Since there's |
| * no way to know whether or not the kernel we're using is one of |
| * the broken ones, the best we can do is to clamp the timeout to |
| * INT64_MAX. This limits the maximum timeout from 584 years to |
| * 292 years - likely not a big deal. |
| */ |
| if (abs_timeout < now) |
| return 0; |
| |
| uint64_t rel_timeout = abs_timeout - now; |
| if (rel_timeout > (uint64_t) INT64_MAX) |
| rel_timeout = INT64_MAX; |
| |
| return rel_timeout; |
| } |
| |
| static struct anv_semaphore *anv_semaphore_ref(struct anv_semaphore *semaphore); |
| static void anv_semaphore_unref(struct anv_device *device, struct anv_semaphore *semaphore); |
| static void anv_semaphore_impl_cleanup(struct anv_device *device, |
| struct anv_semaphore_impl *impl); |
| |
| static void |
| anv_queue_submit_free(struct anv_device *device, |
| struct anv_queue_submit *submit) |
| { |
| const VkAllocationCallbacks *alloc = submit->alloc; |
| |
| for (uint32_t i = 0; i < submit->temporary_semaphore_count; i++) |
| anv_semaphore_impl_cleanup(device, &submit->temporary_semaphores[i]); |
| for (uint32_t i = 0; i < submit->sync_fd_semaphore_count; i++) |
| anv_semaphore_unref(device, submit->sync_fd_semaphores[i]); |
| /* Execbuf does not consume the in_fence. It's our job to close it. */ |
| if (submit->in_fence != -1) |
| close(submit->in_fence); |
| if (submit->out_fence != -1) |
| close(submit->out_fence); |
| vk_free(alloc, submit->fences); |
| vk_free(alloc, submit->temporary_semaphores); |
| vk_free(alloc, submit->wait_timelines); |
| vk_free(alloc, submit->wait_timeline_values); |
| vk_free(alloc, submit->signal_timelines); |
| vk_free(alloc, submit->signal_timeline_values); |
| vk_free(alloc, submit->fence_bos); |
| vk_free(alloc, submit); |
| } |
| |
| static bool |
| anv_queue_submit_ready_locked(struct anv_queue_submit *submit) |
| { |
| for (uint32_t i = 0; i < submit->wait_timeline_count; i++) { |
| if (submit->wait_timeline_values[i] > submit->wait_timelines[i]->highest_pending) |
| return false; |
| } |
| |
| return true; |
| } |
| |
| static VkResult |
| anv_timeline_init(struct anv_device *device, |
| struct anv_timeline *timeline, |
| uint64_t initial_value) |
| { |
| timeline->highest_past = |
| timeline->highest_pending = initial_value; |
| list_inithead(&timeline->points); |
| list_inithead(&timeline->free_points); |
| |
| return VK_SUCCESS; |
| } |
| |
| static void |
| anv_timeline_finish(struct anv_device *device, |
| struct anv_timeline *timeline) |
| { |
| list_for_each_entry_safe(struct anv_timeline_point, point, |
| &timeline->free_points, link) { |
| list_del(&point->link); |
| anv_device_release_bo(device, point->bo); |
| vk_free(&device->vk.alloc, point); |
| } |
| list_for_each_entry_safe(struct anv_timeline_point, point, |
| &timeline->points, link) { |
| list_del(&point->link); |
| anv_device_release_bo(device, point->bo); |
| vk_free(&device->vk.alloc, point); |
| } |
| } |
| |
| static VkResult |
| anv_timeline_add_point_locked(struct anv_device *device, |
| struct anv_timeline *timeline, |
| uint64_t value, |
| struct anv_timeline_point **point) |
| { |
| VkResult result = VK_SUCCESS; |
| |
| if (list_is_empty(&timeline->free_points)) { |
| *point = |
| vk_zalloc(&device->vk.alloc, sizeof(**point), |
| 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); |
| if (!(*point)) |
| result = vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| if (result == VK_SUCCESS) { |
| result = anv_device_alloc_bo(device, 4096, |
| ANV_BO_ALLOC_EXTERNAL | |
| ANV_BO_ALLOC_IMPLICIT_SYNC, |
| 0 /* explicit_address */, |
| &(*point)->bo); |
| if (result != VK_SUCCESS) |
| vk_free(&device->vk.alloc, *point); |
| } |
| } else { |
| *point = list_first_entry(&timeline->free_points, |
| struct anv_timeline_point, link); |
| list_del(&(*point)->link); |
| } |
| |
| if (result == VK_SUCCESS) { |
| (*point)->serial = value; |
| list_addtail(&(*point)->link, &timeline->points); |
| } |
| |
| return result; |
| } |
| |
| static VkResult |
| anv_timeline_gc_locked(struct anv_device *device, |
| struct anv_timeline *timeline) |
| { |
| list_for_each_entry_safe(struct anv_timeline_point, point, |
| &timeline->points, link) { |
| /* timeline->higest_pending is only incremented once submission has |
| * happened. If this point has a greater serial, it means the point |
| * hasn't been submitted yet. |
| */ |
| if (point->serial > timeline->highest_pending) |
| return VK_SUCCESS; |
| |
| /* If someone is waiting on this time point, consider it busy and don't |
| * try to recycle it. There's a slim possibility that it's no longer |
| * busy by the time we look at it but we would be recycling it out from |
| * under a waiter and that can lead to weird races. |
| * |
| * We walk the list in-order so if this time point is still busy so is |
| * every following time point |
| */ |
| assert(point->waiting >= 0); |
| if (point->waiting) |
| return VK_SUCCESS; |
| |
| /* Garbage collect any signaled point. */ |
| VkResult result = anv_device_bo_busy(device, point->bo); |
| if (result == VK_NOT_READY) { |
| /* We walk the list in-order so if this time point is still busy so |
| * is every following time point |
| */ |
| return VK_SUCCESS; |
| } else if (result != VK_SUCCESS) { |
| return result; |
| } |
| |
| assert(timeline->highest_past < point->serial); |
| timeline->highest_past = point->serial; |
| |
| list_del(&point->link); |
| list_add(&point->link, &timeline->free_points); |
| } |
| |
| return VK_SUCCESS; |
| } |
| |
| static VkResult anv_queue_submit_add_fence_bo(struct anv_queue_submit *submit, |
| struct anv_bo *bo, |
| bool signal); |
| |
| static VkResult |
| anv_queue_submit_timeline_locked(struct anv_queue *queue, |
| struct anv_queue_submit *submit) |
| { |
| VkResult result; |
| |
| for (uint32_t i = 0; i < submit->wait_timeline_count; i++) { |
| struct anv_timeline *timeline = submit->wait_timelines[i]; |
| uint64_t wait_value = submit->wait_timeline_values[i]; |
| |
| if (timeline->highest_past >= wait_value) |
| continue; |
| |
| list_for_each_entry(struct anv_timeline_point, point, &timeline->points, link) { |
| if (point->serial < wait_value) |
| continue; |
| result = anv_queue_submit_add_fence_bo(submit, point->bo, false); |
| if (result != VK_SUCCESS) |
| return result; |
| break; |
| } |
| } |
| for (uint32_t i = 0; i < submit->signal_timeline_count; i++) { |
| struct anv_timeline *timeline = submit->signal_timelines[i]; |
| uint64_t signal_value = submit->signal_timeline_values[i]; |
| struct anv_timeline_point *point; |
| |
| result = anv_timeline_add_point_locked(queue->device, timeline, |
| signal_value, &point); |
| if (result != VK_SUCCESS) |
| return result; |
| |
| result = anv_queue_submit_add_fence_bo(submit, point->bo, true); |
| if (result != VK_SUCCESS) |
| return result; |
| } |
| |
| result = anv_queue_execbuf_locked(queue, submit); |
| |
| if (result == VK_SUCCESS) { |
| /* Update the pending values in the timeline objects. */ |
| for (uint32_t i = 0; i < submit->signal_timeline_count; i++) { |
| struct anv_timeline *timeline = submit->signal_timelines[i]; |
| uint64_t signal_value = submit->signal_timeline_values[i]; |
| |
| assert(signal_value > timeline->highest_pending); |
| timeline->highest_pending = signal_value; |
| } |
| |
| /* Update signaled semaphores backed by syncfd. */ |
| for (uint32_t i = 0; i < submit->sync_fd_semaphore_count; i++) { |
| struct anv_semaphore *semaphore = submit->sync_fd_semaphores[i]; |
| /* Out fences can't have temporary state because that would imply |
| * that we imported a sync file and are trying to signal it. |
| */ |
| assert(semaphore->temporary.type == ANV_SEMAPHORE_TYPE_NONE); |
| struct anv_semaphore_impl *impl = &semaphore->permanent; |
| |
| assert(impl->type == ANV_SEMAPHORE_TYPE_SYNC_FILE); |
| impl->fd = dup(submit->out_fence); |
| } |
| } else { |
| /* Unblock any waiter by signaling the points, the application will get |
| * a device lost error code. |
| */ |
| for (uint32_t i = 0; i < submit->signal_timeline_count; i++) { |
| struct anv_timeline *timeline = submit->signal_timelines[i]; |
| uint64_t signal_value = submit->signal_timeline_values[i]; |
| |
| assert(signal_value > timeline->highest_pending); |
| timeline->highest_past = timeline->highest_pending = signal_value; |
| } |
| } |
| |
| return result; |
| } |
| |
| static VkResult |
| anv_queue_submit_deferred_locked(struct anv_queue *queue, uint32_t *advance) |
| { |
| VkResult result = VK_SUCCESS; |
| |
| /* Go through all the queued submissions and submit then until we find one |
| * that's waiting on a point that hasn't materialized yet. |
| */ |
| list_for_each_entry_safe(struct anv_queue_submit, submit, |
| &queue->queued_submits, link) { |
| if (!anv_queue_submit_ready_locked(submit)) |
| break; |
| |
| (*advance)++; |
| list_del(&submit->link); |
| |
| result = anv_queue_submit_timeline_locked(queue, submit); |
| |
| anv_queue_submit_free(queue->device, submit); |
| |
| if (result != VK_SUCCESS) |
| break; |
| } |
| |
| return result; |
| } |
| |
| static VkResult |
| anv_device_submit_deferred_locked(struct anv_device *device) |
| { |
| uint32_t advance = 0; |
| return anv_queue_submit_deferred_locked(&device->queue, &advance); |
| } |
| |
| static VkResult |
| _anv_queue_submit(struct anv_queue *queue, struct anv_queue_submit **_submit, |
| bool flush_queue) |
| { |
| struct anv_queue_submit *submit = *_submit; |
| |
| /* Wait before signal behavior means we might keep alive the |
| * anv_queue_submit object a bit longer, so transfer the ownership to the |
| * anv_queue. |
| */ |
| *_submit = NULL; |
| |
| pthread_mutex_lock(&queue->device->mutex); |
| list_addtail(&submit->link, &queue->queued_submits); |
| VkResult result = anv_device_submit_deferred_locked(queue->device); |
| if (flush_queue) { |
| while (result == VK_SUCCESS && !list_is_empty(&queue->queued_submits)) { |
| int ret = pthread_cond_wait(&queue->device->queue_submit, |
| &queue->device->mutex); |
| if (ret != 0) { |
| result = anv_device_set_lost(queue->device, "wait timeout"); |
| break; |
| } |
| |
| result = anv_device_submit_deferred_locked(queue->device); |
| } |
| } |
| pthread_mutex_unlock(&queue->device->mutex); |
| return result; |
| } |
| |
| VkResult |
| anv_queue_init(struct anv_device *device, struct anv_queue *queue) |
| { |
| vk_object_base_init(&device->vk, &queue->base, VK_OBJECT_TYPE_QUEUE); |
| queue->device = device; |
| queue->flags = 0; |
| |
| list_inithead(&queue->queued_submits); |
| |
| return VK_SUCCESS; |
| } |
| |
| void |
| anv_queue_finish(struct anv_queue *queue) |
| { |
| vk_object_base_finish(&queue->base); |
| } |
| |
| static VkResult |
| anv_queue_submit_add_fence_bo(struct anv_queue_submit *submit, |
| struct anv_bo *bo, |
| bool signal) |
| { |
| if (submit->fence_bo_count >= submit->fence_bo_array_length) { |
| uint32_t new_len = MAX2(submit->fence_bo_array_length * 2, 64); |
| |
| submit->fence_bos = |
| vk_realloc(submit->alloc, |
| submit->fence_bos, new_len * sizeof(*submit->fence_bos), |
| 8, submit->alloc_scope); |
| if (submit->fence_bos == NULL) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| submit->fence_bo_array_length = new_len; |
| } |
| |
| /* Take advantage that anv_bo are allocated at 8 byte alignement so we can |
| * use the lowest bit to store whether this is a BO we need to signal. |
| */ |
| submit->fence_bos[submit->fence_bo_count++] = anv_pack_ptr(bo, 1, signal); |
| |
| return VK_SUCCESS; |
| } |
| |
| static VkResult |
| anv_queue_submit_add_syncobj(struct anv_queue_submit* submit, |
| struct anv_device *device, |
| uint32_t handle, uint32_t flags) |
| { |
| assert(flags != 0); |
| |
| if (submit->fence_count >= submit->fence_array_length) { |
| uint32_t new_len = MAX2(submit->fence_array_length * 2, 64); |
| |
| submit->fences = |
| vk_realloc(submit->alloc, |
| submit->fences, new_len * sizeof(*submit->fences), |
| 8, submit->alloc_scope); |
| if (submit->fences == NULL) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| submit->fence_array_length = new_len; |
| } |
| |
| submit->fences[submit->fence_count++] = (struct drm_i915_gem_exec_fence) { |
| .handle = handle, |
| .flags = flags, |
| }; |
| |
| return VK_SUCCESS; |
| } |
| |
| static VkResult |
| anv_queue_submit_add_sync_fd_fence(struct anv_queue_submit *submit, |
| struct anv_semaphore *semaphore) |
| { |
| if (submit->sync_fd_semaphore_count >= submit->sync_fd_semaphore_array_length) { |
| uint32_t new_len = MAX2(submit->sync_fd_semaphore_array_length * 2, 64); |
| struct anv_semaphore **new_semaphores = |
| vk_realloc(submit->alloc, submit->sync_fd_semaphores, |
| new_len * sizeof(*submit->sync_fd_semaphores), 8, |
| submit->alloc_scope); |
| if (new_semaphores == NULL) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| submit->sync_fd_semaphores = new_semaphores; |
| } |
| |
| submit->sync_fd_semaphores[submit->sync_fd_semaphore_count++] = |
| anv_semaphore_ref(semaphore); |
| submit->need_out_fence = true; |
| |
| return VK_SUCCESS; |
| } |
| |
| static VkResult |
| anv_queue_submit_add_timeline_wait(struct anv_queue_submit* submit, |
| struct anv_device *device, |
| struct anv_timeline *timeline, |
| uint64_t value) |
| { |
| if (submit->wait_timeline_count >= submit->wait_timeline_array_length) { |
| uint32_t new_len = MAX2(submit->wait_timeline_array_length * 2, 64); |
| |
| submit->wait_timelines = |
| vk_realloc(submit->alloc, |
| submit->wait_timelines, new_len * sizeof(*submit->wait_timelines), |
| 8, submit->alloc_scope); |
| if (submit->wait_timelines == NULL) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| submit->wait_timeline_values = |
| vk_realloc(submit->alloc, |
| submit->wait_timeline_values, new_len * sizeof(*submit->wait_timeline_values), |
| 8, submit->alloc_scope); |
| if (submit->wait_timeline_values == NULL) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| submit->wait_timeline_array_length = new_len; |
| } |
| |
| submit->wait_timelines[submit->wait_timeline_count] = timeline; |
| submit->wait_timeline_values[submit->wait_timeline_count] = value; |
| |
| submit->wait_timeline_count++; |
| |
| return VK_SUCCESS; |
| } |
| |
| static VkResult |
| anv_queue_submit_add_timeline_signal(struct anv_queue_submit* submit, |
| struct anv_device *device, |
| struct anv_timeline *timeline, |
| uint64_t value) |
| { |
| assert(timeline->highest_pending < value); |
| |
| if (submit->signal_timeline_count >= submit->signal_timeline_array_length) { |
| uint32_t new_len = MAX2(submit->signal_timeline_array_length * 2, 64); |
| |
| submit->signal_timelines = |
| vk_realloc(submit->alloc, |
| submit->signal_timelines, new_len * sizeof(*submit->signal_timelines), |
| 8, submit->alloc_scope); |
| if (submit->signal_timelines == NULL) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| submit->signal_timeline_values = |
| vk_realloc(submit->alloc, |
| submit->signal_timeline_values, new_len * sizeof(*submit->signal_timeline_values), |
| 8, submit->alloc_scope); |
| if (submit->signal_timeline_values == NULL) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| submit->signal_timeline_array_length = new_len; |
| } |
| |
| submit->signal_timelines[submit->signal_timeline_count] = timeline; |
| submit->signal_timeline_values[submit->signal_timeline_count] = value; |
| |
| submit->signal_timeline_count++; |
| |
| return VK_SUCCESS; |
| } |
| |
| static struct anv_queue_submit * |
| anv_queue_submit_alloc(struct anv_device *device) |
| { |
| const VkAllocationCallbacks *alloc = &device->vk.alloc; |
| VkSystemAllocationScope alloc_scope = VK_SYSTEM_ALLOCATION_SCOPE_DEVICE; |
| |
| struct anv_queue_submit *submit = vk_zalloc(alloc, sizeof(*submit), 8, alloc_scope); |
| if (!submit) |
| return NULL; |
| |
| submit->alloc = alloc; |
| submit->alloc_scope = alloc_scope; |
| submit->in_fence = -1; |
| submit->out_fence = -1; |
| |
| return submit; |
| } |
| |
| VkResult |
| anv_queue_submit_simple_batch(struct anv_queue *queue, |
| struct anv_batch *batch) |
| { |
| if (queue->device->no_hw) |
| return VK_SUCCESS; |
| |
| struct anv_device *device = queue->device; |
| struct anv_queue_submit *submit = anv_queue_submit_alloc(device); |
| if (!submit) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| bool has_syncobj_wait = device->physical->has_syncobj_wait; |
| VkResult result; |
| uint32_t syncobj; |
| struct anv_bo *batch_bo, *sync_bo; |
| |
| if (has_syncobj_wait) { |
| syncobj = anv_gem_syncobj_create(device, 0); |
| if (!syncobj) { |
| result = vk_error(VK_ERROR_OUT_OF_DEVICE_MEMORY); |
| goto err_free_submit; |
| } |
| |
| result = anv_queue_submit_add_syncobj(submit, device, syncobj, |
| I915_EXEC_FENCE_SIGNAL); |
| } else { |
| result = anv_device_alloc_bo(device, 4096, |
| ANV_BO_ALLOC_EXTERNAL | |
| ANV_BO_ALLOC_IMPLICIT_SYNC, |
| 0 /* explicit_address */, |
| &sync_bo); |
| if (result != VK_SUCCESS) |
| goto err_free_submit; |
| |
| result = anv_queue_submit_add_fence_bo(submit, sync_bo, true /* signal */); |
| } |
| |
| if (result != VK_SUCCESS) |
| goto err_destroy_sync_primitive; |
| |
| if (batch) { |
| uint32_t size = align_u32(batch->next - batch->start, 8); |
| result = anv_bo_pool_alloc(&device->batch_bo_pool, size, &batch_bo); |
| if (result != VK_SUCCESS) |
| goto err_destroy_sync_primitive; |
| |
| memcpy(batch_bo->map, batch->start, size); |
| if (!device->info.has_llc) |
| gen_flush_range(batch_bo->map, size); |
| |
| submit->simple_bo = batch_bo; |
| submit->simple_bo_size = size; |
| } |
| |
| result = _anv_queue_submit(queue, &submit, true); |
| |
| if (result == VK_SUCCESS) { |
| if (has_syncobj_wait) { |
| if (anv_gem_syncobj_wait(device, &syncobj, 1, |
| anv_get_absolute_timeout(INT64_MAX), true)) |
| result = anv_device_set_lost(device, "anv_gem_syncobj_wait failed: %m"); |
| anv_gem_syncobj_destroy(device, syncobj); |
| } else { |
| result = anv_device_wait(device, sync_bo, |
| anv_get_relative_timeout(INT64_MAX)); |
| anv_device_release_bo(device, sync_bo); |
| } |
| } |
| |
| if (batch) |
| anv_bo_pool_free(&device->batch_bo_pool, batch_bo); |
| |
| if (submit) |
| anv_queue_submit_free(device, submit); |
| |
| return result; |
| |
| err_destroy_sync_primitive: |
| if (has_syncobj_wait) |
| anv_gem_syncobj_destroy(device, syncobj); |
| else |
| anv_device_release_bo(device, sync_bo); |
| err_free_submit: |
| if (submit) |
| anv_queue_submit_free(device, submit); |
| |
| return result; |
| } |
| |
| /* Transfer ownership of temporary semaphores from the VkSemaphore object to |
| * the anv_queue_submit object. Those temporary semaphores are then freed in |
| * anv_queue_submit_free() once the driver is finished with them. |
| */ |
| static VkResult |
| maybe_transfer_temporary_semaphore(struct anv_queue_submit *submit, |
| struct anv_semaphore *semaphore, |
| struct anv_semaphore_impl **out_impl) |
| { |
| struct anv_semaphore_impl *impl = &semaphore->temporary; |
| |
| if (impl->type == ANV_SEMAPHORE_TYPE_NONE) { |
| *out_impl = &semaphore->permanent; |
| return VK_SUCCESS; |
| } |
| |
| /* BO backed timeline semaphores cannot be temporary. */ |
| assert(impl->type != ANV_SEMAPHORE_TYPE_TIMELINE); |
| |
| /* |
| * There is a requirement to reset semaphore to their permanent state after |
| * submission. From the Vulkan 1.0.53 spec: |
| * |
| * "If the import is temporary, the implementation must restore the |
| * semaphore to its prior permanent state after submitting the next |
| * semaphore wait operation." |
| * |
| * In the case we defer the actual submission to a thread because of the |
| * wait-before-submit behavior required for timeline semaphores, we need to |
| * make copies of the temporary syncobj to ensure they stay alive until we |
| * do the actual execbuffer ioctl. |
| */ |
| if (submit->temporary_semaphore_count >= submit->temporary_semaphore_array_length) { |
| uint32_t new_len = MAX2(submit->temporary_semaphore_array_length * 2, 8); |
| /* Make sure that if the realloc fails, we still have the old semaphore |
| * array around to properly clean things up on failure. |
| */ |
| struct anv_semaphore_impl *new_array = |
| vk_realloc(submit->alloc, |
| submit->temporary_semaphores, |
| new_len * sizeof(*submit->temporary_semaphores), |
| 8, submit->alloc_scope); |
| if (new_array == NULL) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| submit->temporary_semaphores = new_array; |
| submit->temporary_semaphore_array_length = new_len; |
| } |
| |
| /* Copy anv_semaphore_impl into anv_queue_submit. */ |
| submit->temporary_semaphores[submit->temporary_semaphore_count++] = *impl; |
| *out_impl = &submit->temporary_semaphores[submit->temporary_semaphore_count - 1]; |
| |
| /* Clear the incoming semaphore */ |
| impl->type = ANV_SEMAPHORE_TYPE_NONE; |
| |
| return VK_SUCCESS; |
| } |
| |
| static VkResult |
| anv_queue_submit(struct anv_queue *queue, |
| struct anv_cmd_buffer *cmd_buffer, |
| const VkSemaphore *in_semaphores, |
| const uint64_t *in_values, |
| uint32_t num_in_semaphores, |
| const VkSemaphore *out_semaphores, |
| const uint64_t *out_values, |
| uint32_t num_out_semaphores, |
| struct anv_bo *wsi_signal_bo, |
| VkFence _fence) |
| { |
| ANV_FROM_HANDLE(anv_fence, fence, _fence); |
| struct anv_device *device = queue->device; |
| UNUSED struct anv_physical_device *pdevice = device->physical; |
| struct anv_queue_submit *submit = anv_queue_submit_alloc(device); |
| if (!submit) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| submit->cmd_buffer = cmd_buffer; |
| |
| VkResult result = VK_SUCCESS; |
| |
| for (uint32_t i = 0; i < num_in_semaphores; i++) { |
| ANV_FROM_HANDLE(anv_semaphore, semaphore, in_semaphores[i]); |
| struct anv_semaphore_impl *impl; |
| |
| result = maybe_transfer_temporary_semaphore(submit, semaphore, &impl); |
| if (result != VK_SUCCESS) |
| goto error; |
| |
| switch (impl->type) { |
| case ANV_SEMAPHORE_TYPE_BO: |
| assert(!pdevice->has_syncobj); |
| result = anv_queue_submit_add_fence_bo(submit, impl->bo, false /* signal */); |
| if (result != VK_SUCCESS) |
| goto error; |
| break; |
| |
| case ANV_SEMAPHORE_TYPE_WSI_BO: |
| /* When using a window-system buffer as a semaphore, always enable |
| * EXEC_OBJECT_WRITE. This gives us a WaR hazard with the display or |
| * compositor's read of the buffer and enforces that we don't start |
| * rendering until they are finished. This is exactly the |
| * synchronization we want with vkAcquireNextImage. |
| */ |
| result = anv_queue_submit_add_fence_bo(submit, impl->bo, true /* signal */); |
| if (result != VK_SUCCESS) |
| goto error; |
| break; |
| |
| case ANV_SEMAPHORE_TYPE_SYNC_FILE: |
| assert(!pdevice->has_syncobj); |
| if (submit->in_fence == -1) { |
| submit->in_fence = impl->fd; |
| if (submit->in_fence == -1) { |
| result = vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); |
| goto error; |
| } |
| impl->fd = -1; |
| } else { |
| int merge = anv_gem_sync_file_merge(device, submit->in_fence, impl->fd); |
| if (merge == -1) { |
| result = vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); |
| goto error; |
| } |
| close(impl->fd); |
| close(submit->in_fence); |
| impl->fd = -1; |
| submit->in_fence = merge; |
| } |
| break; |
| |
| case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ: { |
| result = anv_queue_submit_add_syncobj(submit, device, |
| impl->syncobj, |
| I915_EXEC_FENCE_WAIT); |
| if (result != VK_SUCCESS) |
| goto error; |
| break; |
| } |
| |
| case ANV_SEMAPHORE_TYPE_TIMELINE: |
| result = anv_queue_submit_add_timeline_wait(submit, device, |
| &impl->timeline, |
| in_values ? in_values[i] : 0); |
| if (result != VK_SUCCESS) |
| goto error; |
| break; |
| |
| default: |
| break; |
| } |
| } |
| |
| for (uint32_t i = 0; i < num_out_semaphores; i++) { |
| ANV_FROM_HANDLE(anv_semaphore, semaphore, out_semaphores[i]); |
| |
| /* Under most circumstances, out fences won't be temporary. However, |
| * the spec does allow it for opaque_fd. From the Vulkan 1.0.53 spec: |
| * |
| * "If the import is temporary, the implementation must restore the |
| * semaphore to its prior permanent state after submitting the next |
| * semaphore wait operation." |
| * |
| * The spec says nothing whatsoever about signal operations on |
| * temporarily imported semaphores so it appears they are allowed. |
| * There are also CTS tests that require this to work. |
| */ |
| struct anv_semaphore_impl *impl = |
| semaphore->temporary.type != ANV_SEMAPHORE_TYPE_NONE ? |
| &semaphore->temporary : &semaphore->permanent; |
| |
| switch (impl->type) { |
| case ANV_SEMAPHORE_TYPE_BO: |
| assert(!pdevice->has_syncobj); |
| result = anv_queue_submit_add_fence_bo(submit, impl->bo, true /* signal */); |
| if (result != VK_SUCCESS) |
| goto error; |
| break; |
| |
| case ANV_SEMAPHORE_TYPE_SYNC_FILE: |
| assert(!pdevice->has_syncobj); |
| result = anv_queue_submit_add_sync_fd_fence(submit, semaphore); |
| if (result != VK_SUCCESS) |
| goto error; |
| break; |
| |
| case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ: { |
| result = anv_queue_submit_add_syncobj(submit, device, impl->syncobj, |
| I915_EXEC_FENCE_SIGNAL); |
| if (result != VK_SUCCESS) |
| goto error; |
| break; |
| } |
| |
| case ANV_SEMAPHORE_TYPE_TIMELINE: |
| result = anv_queue_submit_add_timeline_signal(submit, device, |
| &impl->timeline, |
| out_values ? out_values[i] : 0); |
| if (result != VK_SUCCESS) |
| goto error; |
| break; |
| |
| default: |
| break; |
| } |
| } |
| |
| if (wsi_signal_bo) { |
| result = anv_queue_submit_add_fence_bo(submit, wsi_signal_bo, true /* signal */); |
| if (result != VK_SUCCESS) |
| goto error; |
| } |
| |
| if (fence) { |
| /* Under most circumstances, out fences won't be temporary. However, |
| * the spec does allow it for opaque_fd. From the Vulkan 1.0.53 spec: |
| * |
| * "If the import is temporary, the implementation must restore the |
| * semaphore to its prior permanent state after submitting the next |
| * semaphore wait operation." |
| * |
| * The spec says nothing whatsoever about signal operations on |
| * temporarily imported semaphores so it appears they are allowed. |
| * There are also CTS tests that require this to work. |
| */ |
| struct anv_fence_impl *impl = |
| fence->temporary.type != ANV_FENCE_TYPE_NONE ? |
| &fence->temporary : &fence->permanent; |
| |
| switch (impl->type) { |
| case ANV_FENCE_TYPE_BO: |
| result = anv_queue_submit_add_fence_bo(submit, impl->bo.bo, true /* signal */); |
| if (result != VK_SUCCESS) |
| goto error; |
| break; |
| |
| case ANV_FENCE_TYPE_SYNCOBJ: { |
| /* |
| * For the same reason we reset the signaled binary syncobj above, |
| * also reset the fence's syncobj so that they don't contain a |
| * signaled dma-fence. |
| */ |
| result = anv_queue_submit_add_syncobj(submit, device, impl->syncobj, |
| I915_EXEC_FENCE_SIGNAL); |
| if (result != VK_SUCCESS) |
| goto error; |
| break; |
| } |
| |
| default: |
| unreachable("Invalid fence type"); |
| } |
| } |
| |
| result = _anv_queue_submit(queue, &submit, false); |
| if (result != VK_SUCCESS) |
| goto error; |
| |
| if (fence && fence->permanent.type == ANV_FENCE_TYPE_BO) { |
| /* If we have permanent BO fence, the only type of temporary possible |
| * would be BO_WSI (because BO fences are not shareable). The Vulkan spec |
| * also requires that the fence passed to vkQueueSubmit() be : |
| * |
| * * unsignaled |
| * * not be associated with any other queue command that has not yet |
| * completed execution on that queue |
| * |
| * So the only acceptable type for the temporary is NONE. |
| */ |
| assert(fence->temporary.type == ANV_FENCE_TYPE_NONE); |
| |
| /* Once the execbuf has returned, we need to set the fence state to |
| * SUBMITTED. We can't do this before calling execbuf because |
| * anv_GetFenceStatus does take the global device lock before checking |
| * fence->state. |
| * |
| * We set the fence state to SUBMITTED regardless of whether or not the |
| * execbuf succeeds because we need to ensure that vkWaitForFences() and |
| * vkGetFenceStatus() return a valid result (VK_ERROR_DEVICE_LOST or |
| * VK_SUCCESS) in a finite amount of time even if execbuf fails. |
| */ |
| fence->permanent.bo.state = ANV_BO_FENCE_STATE_SUBMITTED; |
| } |
| |
| error: |
| if (submit) |
| anv_queue_submit_free(device, submit); |
| |
| return result; |
| } |
| |
| VkResult anv_QueueSubmit( |
| VkQueue _queue, |
| uint32_t submitCount, |
| const VkSubmitInfo* pSubmits, |
| VkFence fence) |
| { |
| ANV_FROM_HANDLE(anv_queue, queue, _queue); |
| |
| if (queue->device->no_hw) |
| return VK_SUCCESS; |
| |
| /* Query for device status prior to submitting. Technically, we don't need |
| * to do this. However, if we have a client that's submitting piles of |
| * garbage, we would rather break as early as possible to keep the GPU |
| * hanging contained. If we don't check here, we'll either be waiting for |
| * the kernel to kick us or we'll have to wait until the client waits on a |
| * fence before we actually know whether or not we've hung. |
| */ |
| VkResult result = anv_device_query_status(queue->device); |
| if (result != VK_SUCCESS) |
| return result; |
| |
| if (fence && submitCount == 0) { |
| /* If we don't have any command buffers, we need to submit a dummy |
| * batch to give GEM something to wait on. We could, potentially, |
| * come up with something more efficient but this shouldn't be a |
| * common case. |
| */ |
| result = anv_queue_submit(queue, NULL, NULL, NULL, 0, NULL, NULL, 0, |
| NULL, fence); |
| goto out; |
| } |
| |
| for (uint32_t i = 0; i < submitCount; i++) { |
| /* Fence for this submit. NULL for all but the last one */ |
| VkFence submit_fence = (i == submitCount - 1) ? fence : VK_NULL_HANDLE; |
| |
| const struct wsi_memory_signal_submit_info *mem_signal_info = |
| vk_find_struct_const(pSubmits[i].pNext, |
| WSI_MEMORY_SIGNAL_SUBMIT_INFO_MESA); |
| struct anv_bo *wsi_signal_bo = |
| mem_signal_info && mem_signal_info->memory != VK_NULL_HANDLE ? |
| anv_device_memory_from_handle(mem_signal_info->memory)->bo : NULL; |
| |
| const VkTimelineSemaphoreSubmitInfoKHR *timeline_info = |
| vk_find_struct_const(pSubmits[i].pNext, |
| TIMELINE_SEMAPHORE_SUBMIT_INFO_KHR); |
| const uint64_t *wait_values = |
| timeline_info && timeline_info->waitSemaphoreValueCount ? |
| timeline_info->pWaitSemaphoreValues : NULL; |
| const uint64_t *signal_values = |
| timeline_info && timeline_info->signalSemaphoreValueCount ? |
| timeline_info->pSignalSemaphoreValues : NULL; |
| |
| if (pSubmits[i].commandBufferCount == 0) { |
| /* If we don't have any command buffers, we need to submit a dummy |
| * batch to give GEM something to wait on. We could, potentially, |
| * come up with something more efficient but this shouldn't be a |
| * common case. |
| */ |
| result = anv_queue_submit(queue, NULL, |
| pSubmits[i].pWaitSemaphores, |
| wait_values, |
| pSubmits[i].waitSemaphoreCount, |
| pSubmits[i].pSignalSemaphores, |
| signal_values, |
| pSubmits[i].signalSemaphoreCount, |
| wsi_signal_bo, |
| submit_fence); |
| if (result != VK_SUCCESS) |
| goto out; |
| |
| continue; |
| } |
| |
| for (uint32_t j = 0; j < pSubmits[i].commandBufferCount; j++) { |
| ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, |
| pSubmits[i].pCommandBuffers[j]); |
| assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY); |
| assert(!anv_batch_has_error(&cmd_buffer->batch)); |
| |
| /* Fence for this execbuf. NULL for all but the last one */ |
| VkFence execbuf_fence = |
| (j == pSubmits[i].commandBufferCount - 1) ? |
| submit_fence : VK_NULL_HANDLE; |
| |
| const VkSemaphore *in_semaphores = NULL, *out_semaphores = NULL; |
| const uint64_t *in_values = NULL, *out_values = NULL; |
| uint32_t num_in_semaphores = 0, num_out_semaphores = 0; |
| if (j == 0) { |
| /* Only the first batch gets the in semaphores */ |
| in_semaphores = pSubmits[i].pWaitSemaphores; |
| in_values = wait_values; |
| num_in_semaphores = pSubmits[i].waitSemaphoreCount; |
| } |
| |
| if (j == pSubmits[i].commandBufferCount - 1) { |
| /* Only the last batch gets the out semaphores */ |
| out_semaphores = pSubmits[i].pSignalSemaphores; |
| out_values = signal_values; |
| num_out_semaphores = pSubmits[i].signalSemaphoreCount; |
| } |
| |
| result = anv_queue_submit(queue, cmd_buffer, |
| in_semaphores, in_values, num_in_semaphores, |
| out_semaphores, out_values, num_out_semaphores, |
| wsi_signal_bo, execbuf_fence); |
| if (result != VK_SUCCESS) |
| goto out; |
| } |
| } |
| |
| out: |
| if (result != VK_SUCCESS && result != VK_ERROR_DEVICE_LOST) { |
| /* In the case that something has gone wrong we may end up with an |
| * inconsistent state from which it may not be trivial to recover. |
| * For example, we might have computed address relocations and |
| * any future attempt to re-submit this job will need to know about |
| * this and avoid computing relocation addresses again. |
| * |
| * To avoid this sort of issues, we assume that if something was |
| * wrong during submission we must already be in a really bad situation |
| * anyway (such us being out of memory) and return |
| * VK_ERROR_DEVICE_LOST to ensure that clients do not attempt to |
| * submit the same job again to this device. |
| * |
| * We skip doing this on VK_ERROR_DEVICE_LOST because |
| * anv_device_set_lost() would have been called already by a callee of |
| * anv_queue_submit(). |
| */ |
| result = anv_device_set_lost(queue->device, "vkQueueSubmit() failed"); |
| } |
| |
| return result; |
| } |
| |
| VkResult anv_QueueWaitIdle( |
| VkQueue _queue) |
| { |
| ANV_FROM_HANDLE(anv_queue, queue, _queue); |
| |
| if (anv_device_is_lost(queue->device)) |
| return VK_ERROR_DEVICE_LOST; |
| |
| return anv_queue_submit_simple_batch(queue, NULL); |
| } |
| |
| VkResult anv_CreateFence( |
| VkDevice _device, |
| const VkFenceCreateInfo* pCreateInfo, |
| const VkAllocationCallbacks* pAllocator, |
| VkFence* pFence) |
| { |
| ANV_FROM_HANDLE(anv_device, device, _device); |
| struct anv_fence *fence; |
| |
| assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_FENCE_CREATE_INFO); |
| |
| fence = vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*fence), 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); |
| if (fence == NULL) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| vk_object_base_init(&device->vk, &fence->base, VK_OBJECT_TYPE_FENCE); |
| |
| if (device->physical->has_syncobj_wait) { |
| fence->permanent.type = ANV_FENCE_TYPE_SYNCOBJ; |
| |
| uint32_t create_flags = 0; |
| if (pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT) |
| create_flags |= DRM_SYNCOBJ_CREATE_SIGNALED; |
| |
| fence->permanent.syncobj = anv_gem_syncobj_create(device, create_flags); |
| if (!fence->permanent.syncobj) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| } else { |
| fence->permanent.type = ANV_FENCE_TYPE_BO; |
| |
| VkResult result = anv_bo_pool_alloc(&device->batch_bo_pool, 4096, |
| &fence->permanent.bo.bo); |
| if (result != VK_SUCCESS) |
| return result; |
| |
| if (pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT) { |
| fence->permanent.bo.state = ANV_BO_FENCE_STATE_SIGNALED; |
| } else { |
| fence->permanent.bo.state = ANV_BO_FENCE_STATE_RESET; |
| } |
| } |
| |
| *pFence = anv_fence_to_handle(fence); |
| |
| return VK_SUCCESS; |
| } |
| |
| static void |
| anv_fence_impl_cleanup(struct anv_device *device, |
| struct anv_fence_impl *impl) |
| { |
| switch (impl->type) { |
| case ANV_FENCE_TYPE_NONE: |
| /* Dummy. Nothing to do */ |
| break; |
| |
| case ANV_FENCE_TYPE_BO: |
| anv_bo_pool_free(&device->batch_bo_pool, impl->bo.bo); |
| break; |
| |
| case ANV_FENCE_TYPE_WSI_BO: |
| anv_device_release_bo(device, impl->bo.bo); |
| break; |
| |
| case ANV_FENCE_TYPE_SYNCOBJ: |
| anv_gem_syncobj_destroy(device, impl->syncobj); |
| break; |
| |
| case ANV_FENCE_TYPE_WSI: |
| impl->fence_wsi->destroy(impl->fence_wsi); |
| break; |
| |
| default: |
| unreachable("Invalid fence type"); |
| } |
| |
| impl->type = ANV_FENCE_TYPE_NONE; |
| } |
| |
| void |
| anv_fence_reset_temporary(struct anv_device *device, |
| struct anv_fence *fence) |
| { |
| if (fence->temporary.type == ANV_FENCE_TYPE_NONE) |
| return; |
| |
| anv_fence_impl_cleanup(device, &fence->temporary); |
| } |
| |
| void anv_DestroyFence( |
| VkDevice _device, |
| VkFence _fence, |
| const VkAllocationCallbacks* pAllocator) |
| { |
| ANV_FROM_HANDLE(anv_device, device, _device); |
| ANV_FROM_HANDLE(anv_fence, fence, _fence); |
| |
| if (!fence) |
| return; |
| |
| anv_fence_impl_cleanup(device, &fence->temporary); |
| anv_fence_impl_cleanup(device, &fence->permanent); |
| |
| vk_object_base_finish(&fence->base); |
| vk_free2(&device->vk.alloc, pAllocator, fence); |
| } |
| |
| VkResult anv_ResetFences( |
| VkDevice _device, |
| uint32_t fenceCount, |
| const VkFence* pFences) |
| { |
| ANV_FROM_HANDLE(anv_device, device, _device); |
| |
| for (uint32_t i = 0; i < fenceCount; i++) { |
| ANV_FROM_HANDLE(anv_fence, fence, pFences[i]); |
| |
| /* From the Vulkan 1.0.53 spec: |
| * |
| * "If any member of pFences currently has its payload imported with |
| * temporary permanence, that fence’s prior permanent payload is |
| * first restored. The remaining operations described therefore |
| * operate on the restored payload. |
| */ |
| anv_fence_reset_temporary(device, fence); |
| |
| struct anv_fence_impl *impl = &fence->permanent; |
| |
| switch (impl->type) { |
| case ANV_FENCE_TYPE_BO: |
| impl->bo.state = ANV_BO_FENCE_STATE_RESET; |
| break; |
| |
| case ANV_FENCE_TYPE_SYNCOBJ: |
| anv_gem_syncobj_reset(device, impl->syncobj); |
| break; |
| |
| default: |
| unreachable("Invalid fence type"); |
| } |
| } |
| |
| return VK_SUCCESS; |
| } |
| |
| VkResult anv_GetFenceStatus( |
| VkDevice _device, |
| VkFence _fence) |
| { |
| ANV_FROM_HANDLE(anv_device, device, _device); |
| ANV_FROM_HANDLE(anv_fence, fence, _fence); |
| |
| if (anv_device_is_lost(device)) |
| return VK_ERROR_DEVICE_LOST; |
| |
| struct anv_fence_impl *impl = |
| fence->temporary.type != ANV_FENCE_TYPE_NONE ? |
| &fence->temporary : &fence->permanent; |
| |
| switch (impl->type) { |
| case ANV_FENCE_TYPE_BO: |
| case ANV_FENCE_TYPE_WSI_BO: |
| switch (impl->bo.state) { |
| case ANV_BO_FENCE_STATE_RESET: |
| /* If it hasn't even been sent off to the GPU yet, it's not ready */ |
| return VK_NOT_READY; |
| |
| case ANV_BO_FENCE_STATE_SIGNALED: |
| /* It's been signaled, return success */ |
| return VK_SUCCESS; |
| |
| case ANV_BO_FENCE_STATE_SUBMITTED: { |
| VkResult result = anv_device_bo_busy(device, impl->bo.bo); |
| if (result == VK_SUCCESS) { |
| impl->bo.state = ANV_BO_FENCE_STATE_SIGNALED; |
| return VK_SUCCESS; |
| } else { |
| return result; |
| } |
| } |
| default: |
| unreachable("Invalid fence status"); |
| } |
| |
| case ANV_FENCE_TYPE_SYNCOBJ: { |
| int ret = anv_gem_syncobj_wait(device, &impl->syncobj, 1, 0, true); |
| if (ret == -1) { |
| if (errno == ETIME) { |
| return VK_NOT_READY; |
| } else { |
| /* We don't know the real error. */ |
| return anv_device_set_lost(device, "drm_syncobj_wait failed: %m"); |
| } |
| } else { |
| return VK_SUCCESS; |
| } |
| } |
| |
| default: |
| unreachable("Invalid fence type"); |
| } |
| } |
| |
| static VkResult |
| anv_wait_for_syncobj_fences(struct anv_device *device, |
| uint32_t fenceCount, |
| const VkFence *pFences, |
| bool waitAll, |
| uint64_t abs_timeout_ns) |
| { |
| uint32_t *syncobjs = vk_zalloc(&device->vk.alloc, |
| sizeof(*syncobjs) * fenceCount, 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); |
| if (!syncobjs) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| for (uint32_t i = 0; i < fenceCount; i++) { |
| ANV_FROM_HANDLE(anv_fence, fence, pFences[i]); |
| assert(fence->permanent.type == ANV_FENCE_TYPE_SYNCOBJ); |
| |
| struct anv_fence_impl *impl = |
| fence->temporary.type != ANV_FENCE_TYPE_NONE ? |
| &fence->temporary : &fence->permanent; |
| |
| assert(impl->type == ANV_FENCE_TYPE_SYNCOBJ); |
| syncobjs[i] = impl->syncobj; |
| } |
| |
| /* The gem_syncobj_wait ioctl may return early due to an inherent |
| * limitation in the way it computes timeouts. Loop until we've actually |
| * passed the timeout. |
| */ |
| int ret; |
| do { |
| ret = anv_gem_syncobj_wait(device, syncobjs, fenceCount, |
| abs_timeout_ns, waitAll); |
| } while (ret == -1 && errno == ETIME && anv_gettime_ns() < abs_timeout_ns); |
| |
| vk_free(&device->vk.alloc, syncobjs); |
| |
| if (ret == -1) { |
| if (errno == ETIME) { |
| return VK_TIMEOUT; |
| } else { |
| /* We don't know the real error. */ |
| return anv_device_set_lost(device, "drm_syncobj_wait failed: %m"); |
| } |
| } else { |
| return VK_SUCCESS; |
| } |
| } |
| |
| static VkResult |
| anv_wait_for_bo_fences(struct anv_device *device, |
| uint32_t fenceCount, |
| const VkFence *pFences, |
| bool waitAll, |
| uint64_t abs_timeout_ns) |
| { |
| VkResult result = VK_SUCCESS; |
| uint32_t pending_fences = fenceCount; |
| while (pending_fences) { |
| pending_fences = 0; |
| bool signaled_fences = false; |
| for (uint32_t i = 0; i < fenceCount; i++) { |
| ANV_FROM_HANDLE(anv_fence, fence, pFences[i]); |
| |
| struct anv_fence_impl *impl = |
| fence->temporary.type != ANV_FENCE_TYPE_NONE ? |
| &fence->temporary : &fence->permanent; |
| assert(impl->type == ANV_FENCE_TYPE_BO || |
| impl->type == ANV_FENCE_TYPE_WSI_BO); |
| |
| switch (impl->bo.state) { |
| case ANV_BO_FENCE_STATE_RESET: |
| /* This fence hasn't been submitted yet, we'll catch it the next |
| * time around. Yes, this may mean we dead-loop but, short of |
| * lots of locking and a condition variable, there's not much that |
| * we can do about that. |
| */ |
| pending_fences++; |
| continue; |
| |
| case ANV_BO_FENCE_STATE_SIGNALED: |
| /* This fence is not pending. If waitAll isn't set, we can return |
| * early. Otherwise, we have to keep going. |
| */ |
| if (!waitAll) { |
| result = VK_SUCCESS; |
| goto done; |
| } |
| continue; |
| |
| case ANV_BO_FENCE_STATE_SUBMITTED: |
| /* These are the fences we really care about. Go ahead and wait |
| * on it until we hit a timeout. |
| */ |
| result = anv_device_wait(device, impl->bo.bo, |
| anv_get_relative_timeout(abs_timeout_ns)); |
| switch (result) { |
| case VK_SUCCESS: |
| impl->bo.state = ANV_BO_FENCE_STATE_SIGNALED; |
| signaled_fences = true; |
| if (!waitAll) |
| goto done; |
| break; |
| |
| case VK_TIMEOUT: |
| goto done; |
| |
| default: |
| return result; |
| } |
| } |
| } |
| |
| if (pending_fences && !signaled_fences) { |
| /* If we've hit this then someone decided to vkWaitForFences before |
| * they've actually submitted any of them to a queue. This is a |
| * fairly pessimal case, so it's ok to lock here and use a standard |
| * pthreads condition variable. |
| */ |
| pthread_mutex_lock(&device->mutex); |
| |
| /* It's possible that some of the fences have changed state since the |
| * last time we checked. Now that we have the lock, check for |
| * pending fences again and don't wait if it's changed. |
| */ |
| uint32_t now_pending_fences = 0; |
| for (uint32_t i = 0; i < fenceCount; i++) { |
| ANV_FROM_HANDLE(anv_fence, fence, pFences[i]); |
| if (fence->permanent.bo.state == ANV_BO_FENCE_STATE_RESET) |
| now_pending_fences++; |
| } |
| assert(now_pending_fences <= pending_fences); |
| |
| if (now_pending_fences == pending_fences) { |
| struct timespec abstime = { |
| .tv_sec = abs_timeout_ns / NSEC_PER_SEC, |
| .tv_nsec = abs_timeout_ns % NSEC_PER_SEC, |
| }; |
| |
| ASSERTED int ret; |
| ret = pthread_cond_timedwait(&device->queue_submit, |
| &device->mutex, &abstime); |
| assert(ret != EINVAL); |
| if (anv_gettime_ns() >= abs_timeout_ns) { |
| pthread_mutex_unlock(&device->mutex); |
| result = VK_TIMEOUT; |
| goto done; |
| } |
| } |
| |
| pthread_mutex_unlock(&device->mutex); |
| } |
| } |
| |
| done: |
| if (anv_device_is_lost(device)) |
| return VK_ERROR_DEVICE_LOST; |
| |
| return result; |
| } |
| |
| static VkResult |
| anv_wait_for_wsi_fence(struct anv_device *device, |
| struct anv_fence_impl *impl, |
| uint64_t abs_timeout) |
| { |
| return impl->fence_wsi->wait(impl->fence_wsi, abs_timeout); |
| } |
| |
| static VkResult |
| anv_wait_for_fences(struct anv_device *device, |
| uint32_t fenceCount, |
| const VkFence *pFences, |
| bool waitAll, |
| uint64_t abs_timeout) |
| { |
| VkResult result = VK_SUCCESS; |
| |
| if (fenceCount <= 1 || waitAll) { |
| for (uint32_t i = 0; i < fenceCount; i++) { |
| ANV_FROM_HANDLE(anv_fence, fence, pFences[i]); |
| struct anv_fence_impl *impl = |
| fence->temporary.type != ANV_FENCE_TYPE_NONE ? |
| &fence->temporary : &fence->permanent; |
| |
| switch (impl->type) { |
| case ANV_FENCE_TYPE_BO: |
| case ANV_FENCE_TYPE_WSI_BO: |
| result = anv_wait_for_bo_fences(device, 1, &pFences[i], |
| true, abs_timeout); |
| break; |
| case ANV_FENCE_TYPE_SYNCOBJ: |
| result = anv_wait_for_syncobj_fences(device, 1, &pFences[i], |
| true, abs_timeout); |
| break; |
| case ANV_FENCE_TYPE_WSI: |
| result = anv_wait_for_wsi_fence(device, impl, abs_timeout); |
| break; |
| case ANV_FENCE_TYPE_NONE: |
| result = VK_SUCCESS; |
| break; |
| } |
| if (result != VK_SUCCESS) |
| return result; |
| } |
| } else { |
| do { |
| for (uint32_t i = 0; i < fenceCount; i++) { |
| if (anv_wait_for_fences(device, 1, &pFences[i], true, 0) == VK_SUCCESS) |
| return VK_SUCCESS; |
| } |
| } while (anv_gettime_ns() < abs_timeout); |
| result = VK_TIMEOUT; |
| } |
| return result; |
| } |
| |
| static bool anv_all_fences_syncobj(uint32_t fenceCount, const VkFence *pFences) |
| { |
| for (uint32_t i = 0; i < fenceCount; ++i) { |
| ANV_FROM_HANDLE(anv_fence, fence, pFences[i]); |
| struct anv_fence_impl *impl = |
| fence->temporary.type != ANV_FENCE_TYPE_NONE ? |
| &fence->temporary : &fence->permanent; |
| if (impl->type != ANV_FENCE_TYPE_SYNCOBJ) |
| return false; |
| } |
| return true; |
| } |
| |
| static bool anv_all_fences_bo(uint32_t fenceCount, const VkFence *pFences) |
| { |
| for (uint32_t i = 0; i < fenceCount; ++i) { |
| ANV_FROM_HANDLE(anv_fence, fence, pFences[i]); |
| struct anv_fence_impl *impl = |
| fence->temporary.type != ANV_FENCE_TYPE_NONE ? |
| &fence->temporary : &fence->permanent; |
| if (impl->type != ANV_FENCE_TYPE_BO && |
| impl->type != ANV_FENCE_TYPE_WSI_BO) |
| return false; |
| } |
| return true; |
| } |
| |
| VkResult anv_WaitForFences( |
| VkDevice _device, |
| uint32_t fenceCount, |
| const VkFence* pFences, |
| VkBool32 waitAll, |
| uint64_t timeout) |
| { |
| ANV_FROM_HANDLE(anv_device, device, _device); |
| |
| if (device->no_hw) |
| return VK_SUCCESS; |
| |
| if (anv_device_is_lost(device)) |
| return VK_ERROR_DEVICE_LOST; |
| |
| uint64_t abs_timeout = anv_get_absolute_timeout(timeout); |
| if (anv_all_fences_syncobj(fenceCount, pFences)) { |
| return anv_wait_for_syncobj_fences(device, fenceCount, pFences, |
| waitAll, abs_timeout); |
| } else if (anv_all_fences_bo(fenceCount, pFences)) { |
| return anv_wait_for_bo_fences(device, fenceCount, pFences, |
| waitAll, abs_timeout); |
| } else { |
| return anv_wait_for_fences(device, fenceCount, pFences, |
| waitAll, abs_timeout); |
| } |
| } |
| |
| void anv_GetPhysicalDeviceExternalFenceProperties( |
| VkPhysicalDevice physicalDevice, |
| const VkPhysicalDeviceExternalFenceInfo* pExternalFenceInfo, |
| VkExternalFenceProperties* pExternalFenceProperties) |
| { |
| ANV_FROM_HANDLE(anv_physical_device, device, physicalDevice); |
| |
| switch (pExternalFenceInfo->handleType) { |
| case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT: |
| case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT: |
| if (device->has_syncobj_wait) { |
| pExternalFenceProperties->exportFromImportedHandleTypes = |
| VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT | |
| VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT; |
| pExternalFenceProperties->compatibleHandleTypes = |
| VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT | |
| VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT; |
| pExternalFenceProperties->externalFenceFeatures = |
| VK_EXTERNAL_FENCE_FEATURE_EXPORTABLE_BIT | |
| VK_EXTERNAL_FENCE_FEATURE_IMPORTABLE_BIT; |
| return; |
| } |
| break; |
| |
| default: |
| break; |
| } |
| |
| pExternalFenceProperties->exportFromImportedHandleTypes = 0; |
| pExternalFenceProperties->compatibleHandleTypes = 0; |
| pExternalFenceProperties->externalFenceFeatures = 0; |
| } |
| |
| VkResult anv_ImportFenceFdKHR( |
| VkDevice _device, |
| const VkImportFenceFdInfoKHR* pImportFenceFdInfo) |
| { |
| ANV_FROM_HANDLE(anv_device, device, _device); |
| ANV_FROM_HANDLE(anv_fence, fence, pImportFenceFdInfo->fence); |
| int fd = pImportFenceFdInfo->fd; |
| |
| assert(pImportFenceFdInfo->sType == |
| VK_STRUCTURE_TYPE_IMPORT_FENCE_FD_INFO_KHR); |
| |
| struct anv_fence_impl new_impl = { |
| .type = ANV_FENCE_TYPE_NONE, |
| }; |
| |
| switch (pImportFenceFdInfo->handleType) { |
| case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT: |
| new_impl.type = ANV_FENCE_TYPE_SYNCOBJ; |
| |
| new_impl.syncobj = anv_gem_syncobj_fd_to_handle(device, fd); |
| if (!new_impl.syncobj) |
| return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); |
| |
| break; |
| |
| case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT: |
| /* Sync files are a bit tricky. Because we want to continue using the |
| * syncobj implementation of WaitForFences, we don't use the sync file |
| * directly but instead import it into a syncobj. |
| */ |
| new_impl.type = ANV_FENCE_TYPE_SYNCOBJ; |
| |
| new_impl.syncobj = anv_gem_syncobj_create(device, 0); |
| if (!new_impl.syncobj) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| if (anv_gem_syncobj_import_sync_file(device, new_impl.syncobj, fd)) { |
| anv_gem_syncobj_destroy(device, new_impl.syncobj); |
| return vk_errorf(device, NULL, VK_ERROR_INVALID_EXTERNAL_HANDLE, |
| "syncobj sync file import failed: %m"); |
| } |
| break; |
| |
| default: |
| return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); |
| } |
| |
| /* From the Vulkan 1.0.53 spec: |
| * |
| * "Importing a fence payload from a file descriptor transfers |
| * ownership of the file descriptor from the application to the |
| * Vulkan implementation. The application must not perform any |
| * operations on the file descriptor after a successful import." |
| * |
| * If the import fails, we leave the file descriptor open. |
| */ |
| close(fd); |
| |
| if (pImportFenceFdInfo->flags & VK_FENCE_IMPORT_TEMPORARY_BIT) { |
| anv_fence_impl_cleanup(device, &fence->temporary); |
| fence->temporary = new_impl; |
| } else { |
| anv_fence_impl_cleanup(device, &fence->permanent); |
| fence->permanent = new_impl; |
| } |
| |
| return VK_SUCCESS; |
| } |
| |
| VkResult anv_GetFenceFdKHR( |
| VkDevice _device, |
| const VkFenceGetFdInfoKHR* pGetFdInfo, |
| int* pFd) |
| { |
| ANV_FROM_HANDLE(anv_device, device, _device); |
| ANV_FROM_HANDLE(anv_fence, fence, pGetFdInfo->fence); |
| |
| assert(pGetFdInfo->sType == VK_STRUCTURE_TYPE_FENCE_GET_FD_INFO_KHR); |
| |
| struct anv_fence_impl *impl = |
| fence->temporary.type != ANV_FENCE_TYPE_NONE ? |
| &fence->temporary : &fence->permanent; |
| |
| assert(impl->type == ANV_FENCE_TYPE_SYNCOBJ); |
| switch (pGetFdInfo->handleType) { |
| case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT: { |
| int fd = anv_gem_syncobj_handle_to_fd(device, impl->syncobj); |
| if (fd < 0) |
| return vk_error(VK_ERROR_TOO_MANY_OBJECTS); |
| |
| *pFd = fd; |
| break; |
| } |
| |
| case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT: { |
| int fd = anv_gem_syncobj_export_sync_file(device, impl->syncobj); |
| if (fd < 0) |
| return vk_error(VK_ERROR_TOO_MANY_OBJECTS); |
| |
| *pFd = fd; |
| break; |
| } |
| |
| default: |
| unreachable("Invalid fence export handle type"); |
| } |
| |
| /* From the Vulkan 1.0.53 spec: |
| * |
| * "Export operations have the same transference as the specified handle |
| * type’s import operations. [...] If the fence was using a |
| * temporarily imported payload, the fence’s prior permanent payload |
| * will be restored. |
| */ |
| if (impl == &fence->temporary) |
| anv_fence_impl_cleanup(device, impl); |
| |
| return VK_SUCCESS; |
| } |
| |
| // Queue semaphore functions |
| |
| static VkSemaphoreTypeKHR |
| get_semaphore_type(const void *pNext, uint64_t *initial_value) |
| { |
| const VkSemaphoreTypeCreateInfoKHR *type_info = |
| vk_find_struct_const(pNext, SEMAPHORE_TYPE_CREATE_INFO_KHR); |
| |
| if (!type_info) |
| return VK_SEMAPHORE_TYPE_BINARY_KHR; |
| |
| if (initial_value) |
| *initial_value = type_info->initialValue; |
| return type_info->semaphoreType; |
| } |
| |
| static VkResult |
| binary_semaphore_create(struct anv_device *device, |
| struct anv_semaphore_impl *impl, |
| bool exportable) |
| { |
| if (device->physical->has_syncobj) { |
| impl->type = ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ; |
| impl->syncobj = anv_gem_syncobj_create(device, 0); |
| if (!impl->syncobj) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| return VK_SUCCESS; |
| } else { |
| impl->type = ANV_SEMAPHORE_TYPE_BO; |
| VkResult result = |
| anv_device_alloc_bo(device, 4096, |
| ANV_BO_ALLOC_EXTERNAL | |
| ANV_BO_ALLOC_IMPLICIT_SYNC, |
| 0 /* explicit_address */, |
| &impl->bo); |
| /* If we're going to use this as a fence, we need to *not* have the |
| * EXEC_OBJECT_ASYNC bit set. |
| */ |
| assert(!(impl->bo->flags & EXEC_OBJECT_ASYNC)); |
| return result; |
| } |
| } |
| |
| static VkResult |
| timeline_semaphore_create(struct anv_device *device, |
| struct anv_semaphore_impl *impl, |
| uint64_t initial_value) |
| { |
| impl->type = ANV_SEMAPHORE_TYPE_TIMELINE; |
| anv_timeline_init(device, &impl->timeline, initial_value); |
| return VK_SUCCESS; |
| } |
| |
| VkResult anv_CreateSemaphore( |
| VkDevice _device, |
| const VkSemaphoreCreateInfo* pCreateInfo, |
| const VkAllocationCallbacks* pAllocator, |
| VkSemaphore* pSemaphore) |
| { |
| ANV_FROM_HANDLE(anv_device, device, _device); |
| struct anv_semaphore *semaphore; |
| |
| assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO); |
| |
| uint64_t timeline_value = 0; |
| VkSemaphoreTypeKHR sem_type = get_semaphore_type(pCreateInfo->pNext, &timeline_value); |
| |
| semaphore = vk_alloc(&device->vk.alloc, sizeof(*semaphore), 8, |
| VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); |
| if (semaphore == NULL) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| vk_object_base_init(&device->vk, &semaphore->base, VK_OBJECT_TYPE_SEMAPHORE); |
| |
| p_atomic_set(&semaphore->refcount, 1); |
| |
| const VkExportSemaphoreCreateInfo *export = |
| vk_find_struct_const(pCreateInfo->pNext, EXPORT_SEMAPHORE_CREATE_INFO); |
| VkExternalSemaphoreHandleTypeFlags handleTypes = |
| export ? export->handleTypes : 0; |
| VkResult result; |
| |
| if (handleTypes == 0) { |
| if (sem_type == VK_SEMAPHORE_TYPE_BINARY_KHR) |
| result = binary_semaphore_create(device, &semaphore->permanent, false); |
| else |
| result = timeline_semaphore_create(device, &semaphore->permanent, timeline_value); |
| if (result != VK_SUCCESS) { |
| vk_free2(&device->vk.alloc, pAllocator, semaphore); |
| return result; |
| } |
| } else if (handleTypes & VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT) { |
| assert(handleTypes == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT); |
| assert(sem_type == VK_SEMAPHORE_TYPE_BINARY_KHR); |
| result = binary_semaphore_create(device, &semaphore->permanent, true); |
| if (result != VK_SUCCESS) { |
| vk_free2(&device->vk.alloc, pAllocator, semaphore); |
| return result; |
| } |
| } else if (handleTypes & VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT) { |
| assert(handleTypes == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT); |
| assert(sem_type == VK_SEMAPHORE_TYPE_BINARY_KHR); |
| if (device->physical->has_syncobj) { |
| semaphore->permanent.type = ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ; |
| semaphore->permanent.syncobj = anv_gem_syncobj_create(device, 0); |
| if (!semaphore->permanent.syncobj) { |
| vk_free2(&device->vk.alloc, pAllocator, semaphore); |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| } |
| } else { |
| semaphore->permanent.type = ANV_SEMAPHORE_TYPE_SYNC_FILE; |
| semaphore->permanent.fd = -1; |
| } |
| } else { |
| assert(!"Unknown handle type"); |
| vk_free2(&device->vk.alloc, pAllocator, semaphore); |
| return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); |
| } |
| |
| semaphore->temporary.type = ANV_SEMAPHORE_TYPE_NONE; |
| |
| *pSemaphore = anv_semaphore_to_handle(semaphore); |
| |
| return VK_SUCCESS; |
| } |
| |
| static void |
| anv_semaphore_impl_cleanup(struct anv_device *device, |
| struct anv_semaphore_impl *impl) |
| { |
| switch (impl->type) { |
| case ANV_SEMAPHORE_TYPE_NONE: |
| case ANV_SEMAPHORE_TYPE_DUMMY: |
| /* Dummy. Nothing to do */ |
| break; |
| |
| case ANV_SEMAPHORE_TYPE_BO: |
| case ANV_SEMAPHORE_TYPE_WSI_BO: |
| anv_device_release_bo(device, impl->bo); |
| break; |
| |
| case ANV_SEMAPHORE_TYPE_SYNC_FILE: |
| if (impl->fd >= 0) |
| close(impl->fd); |
| break; |
| |
| case ANV_SEMAPHORE_TYPE_TIMELINE: |
| anv_timeline_finish(device, &impl->timeline); |
| break; |
| |
| case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ: |
| anv_gem_syncobj_destroy(device, impl->syncobj); |
| break; |
| |
| default: |
| unreachable("Invalid semaphore type"); |
| } |
| |
| impl->type = ANV_SEMAPHORE_TYPE_NONE; |
| } |
| |
| void |
| anv_semaphore_reset_temporary(struct anv_device *device, |
| struct anv_semaphore *semaphore) |
| { |
| if (semaphore->temporary.type == ANV_SEMAPHORE_TYPE_NONE) |
| return; |
| |
| anv_semaphore_impl_cleanup(device, &semaphore->temporary); |
| } |
| |
| static struct anv_semaphore * |
| anv_semaphore_ref(struct anv_semaphore *semaphore) |
| { |
| assert(semaphore->refcount); |
| p_atomic_inc(&semaphore->refcount); |
| return semaphore; |
| } |
| |
| static void |
| anv_semaphore_unref(struct anv_device *device, struct anv_semaphore *semaphore) |
| { |
| if (!p_atomic_dec_zero(&semaphore->refcount)) |
| return; |
| |
| anv_semaphore_impl_cleanup(device, &semaphore->temporary); |
| anv_semaphore_impl_cleanup(device, &semaphore->permanent); |
| |
| vk_object_base_finish(&semaphore->base); |
| vk_free(&device->vk.alloc, semaphore); |
| } |
| |
| void anv_DestroySemaphore( |
| VkDevice _device, |
| VkSemaphore _semaphore, |
| const VkAllocationCallbacks* pAllocator) |
| { |
| ANV_FROM_HANDLE(anv_device, device, _device); |
| ANV_FROM_HANDLE(anv_semaphore, semaphore, _semaphore); |
| |
| if (semaphore == NULL) |
| return; |
| |
| anv_semaphore_unref(device, semaphore); |
| } |
| |
| void anv_GetPhysicalDeviceExternalSemaphoreProperties( |
| VkPhysicalDevice physicalDevice, |
| const VkPhysicalDeviceExternalSemaphoreInfo* pExternalSemaphoreInfo, |
| VkExternalSemaphoreProperties* pExternalSemaphoreProperties) |
| { |
| ANV_FROM_HANDLE(anv_physical_device, device, physicalDevice); |
| |
| VkSemaphoreTypeKHR sem_type = |
| get_semaphore_type(pExternalSemaphoreInfo->pNext, NULL); |
| |
| switch (pExternalSemaphoreInfo->handleType) { |
| case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT: |
| /* Timeline semaphores are not exportable. */ |
| if (sem_type == VK_SEMAPHORE_TYPE_TIMELINE_KHR) |
| break; |
| pExternalSemaphoreProperties->exportFromImportedHandleTypes = |
| VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; |
| pExternalSemaphoreProperties->compatibleHandleTypes = |
| VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; |
| pExternalSemaphoreProperties->externalSemaphoreFeatures = |
| VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT | |
| VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT; |
| return; |
| |
| case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT: |
| if (sem_type == VK_SEMAPHORE_TYPE_TIMELINE_KHR) |
| break; |
| if (!device->has_exec_fence) |
| break; |
| pExternalSemaphoreProperties->exportFromImportedHandleTypes = |
| VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT; |
| pExternalSemaphoreProperties->compatibleHandleTypes = |
| VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT; |
| pExternalSemaphoreProperties->externalSemaphoreFeatures = |
| VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT | |
| VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT; |
| return; |
| |
| default: |
| break; |
| } |
| |
| pExternalSemaphoreProperties->exportFromImportedHandleTypes = 0; |
| pExternalSemaphoreProperties->compatibleHandleTypes = 0; |
| pExternalSemaphoreProperties->externalSemaphoreFeatures = 0; |
| } |
| |
| VkResult anv_ImportSemaphoreFdKHR( |
| VkDevice _device, |
| const VkImportSemaphoreFdInfoKHR* pImportSemaphoreFdInfo) |
| { |
| ANV_FROM_HANDLE(anv_device, device, _device); |
| ANV_FROM_HANDLE(anv_semaphore, semaphore, pImportSemaphoreFdInfo->semaphore); |
| int fd = pImportSemaphoreFdInfo->fd; |
| |
| struct anv_semaphore_impl new_impl = { |
| .type = ANV_SEMAPHORE_TYPE_NONE, |
| }; |
| |
| switch (pImportSemaphoreFdInfo->handleType) { |
| case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT: |
| if (device->physical->has_syncobj) { |
| new_impl.type = ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ; |
| |
| new_impl.syncobj = anv_gem_syncobj_fd_to_handle(device, fd); |
| if (!new_impl.syncobj) |
| return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); |
| } else { |
| new_impl.type = ANV_SEMAPHORE_TYPE_BO; |
| |
| VkResult result = anv_device_import_bo(device, fd, |
| ANV_BO_ALLOC_EXTERNAL | |
| ANV_BO_ALLOC_IMPLICIT_SYNC, |
| 0 /* client_address */, |
| &new_impl.bo); |
| if (result != VK_SUCCESS) |
| return result; |
| |
| if (new_impl.bo->size < 4096) { |
| anv_device_release_bo(device, new_impl.bo); |
| return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); |
| } |
| |
| /* If we're going to use this as a fence, we need to *not* have the |
| * EXEC_OBJECT_ASYNC bit set. |
| */ |
| assert(!(new_impl.bo->flags & EXEC_OBJECT_ASYNC)); |
| } |
| |
| /* From the Vulkan spec: |
| * |
| * "Importing semaphore state from a file descriptor transfers |
| * ownership of the file descriptor from the application to the |
| * Vulkan implementation. The application must not perform any |
| * operations on the file descriptor after a successful import." |
| * |
| * If the import fails, we leave the file descriptor open. |
| */ |
| close(fd); |
| break; |
| |
| case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT: |
| if (device->physical->has_syncobj) { |
| new_impl = (struct anv_semaphore_impl) { |
| .type = ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ, |
| .syncobj = anv_gem_syncobj_create(device, 0), |
| }; |
| if (!new_impl.syncobj) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| if (anv_gem_syncobj_import_sync_file(device, new_impl.syncobj, fd)) { |
| anv_gem_syncobj_destroy(device, new_impl.syncobj); |
| return vk_errorf(device, NULL, VK_ERROR_INVALID_EXTERNAL_HANDLE, |
| "syncobj sync file import failed: %m"); |
| } |
| /* Ownership of the FD is transfered to Anv. Since we don't need it |
| * anymore because the associated fence has been put into a syncobj, |
| * we must close the FD. |
| */ |
| close(fd); |
| } else { |
| new_impl = (struct anv_semaphore_impl) { |
| .type = ANV_SEMAPHORE_TYPE_SYNC_FILE, |
| .fd = fd, |
| }; |
| } |
| break; |
| |
| default: |
| return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); |
| } |
| |
| if (pImportSemaphoreFdInfo->flags & VK_SEMAPHORE_IMPORT_TEMPORARY_BIT) { |
| anv_semaphore_impl_cleanup(device, &semaphore->temporary); |
| semaphore->temporary = new_impl; |
| } else { |
| anv_semaphore_impl_cleanup(device, &semaphore->permanent); |
| semaphore->permanent = new_impl; |
| } |
| |
| return VK_SUCCESS; |
| } |
| |
| VkResult anv_GetSemaphoreFdKHR( |
| VkDevice _device, |
| const VkSemaphoreGetFdInfoKHR* pGetFdInfo, |
| int* pFd) |
| { |
| ANV_FROM_HANDLE(anv_device, device, _device); |
| ANV_FROM_HANDLE(anv_semaphore, semaphore, pGetFdInfo->semaphore); |
| VkResult result; |
| int fd; |
| |
| assert(pGetFdInfo->sType == VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR); |
| |
| struct anv_semaphore_impl *impl = |
| semaphore->temporary.type != ANV_SEMAPHORE_TYPE_NONE ? |
| &semaphore->temporary : &semaphore->permanent; |
| |
| switch (impl->type) { |
| case ANV_SEMAPHORE_TYPE_BO: |
| result = anv_device_export_bo(device, impl->bo, pFd); |
| if (result != VK_SUCCESS) |
| return result; |
| break; |
| |
| case ANV_SEMAPHORE_TYPE_SYNC_FILE: { |
| /* There's a potential race here with vkQueueSubmit if you are trying |
| * to export a semaphore Fd while the queue submit is still happening. |
| * This can happen if we see all dependencies get resolved via timeline |
| * semaphore waits completing before the execbuf completes and we |
| * process the resulting out fence. To work around this, take a lock |
| * around grabbing the fd. |
| */ |
| pthread_mutex_lock(&device->mutex); |
| |
| /* From the Vulkan 1.0.53 spec: |
| * |
| * "...exporting a semaphore payload to a handle with copy |
| * transference has the same side effects on the source |
| * semaphore’s payload as executing a semaphore wait operation." |
| * |
| * In other words, it may still be a SYNC_FD semaphore, but it's now |
| * considered to have been waited on and no longer has a sync file |
| * attached. |
| */ |
| int fd = impl->fd; |
| impl->fd = -1; |
| |
| pthread_mutex_unlock(&device->mutex); |
| |
| /* There are two reasons why this could happen: |
| * |
| * 1) The user is trying to export without submitting something that |
| * signals the semaphore. If this is the case, it's their bug so |
| * what we return here doesn't matter. |
| * |
| * 2) The kernel didn't give us a file descriptor. The most likely |
| * reason for this is running out of file descriptors. |
| */ |
| if (fd < 0) |
| return vk_error(VK_ERROR_TOO_MANY_OBJECTS); |
| |
| *pFd = fd; |
| return VK_SUCCESS; |
| } |
| |
| case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ: |
| if (pGetFdInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT) |
| fd = anv_gem_syncobj_export_sync_file(device, impl->syncobj); |
| else { |
| assert(pGetFdInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT); |
| fd = anv_gem_syncobj_handle_to_fd(device, impl->syncobj); |
| } |
| if (fd < 0) |
| return vk_error(VK_ERROR_TOO_MANY_OBJECTS); |
| *pFd = fd; |
| break; |
| |
| default: |
| return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); |
| } |
| |
| /* From the Vulkan 1.0.53 spec: |
| * |
| * "Export operations have the same transference as the specified handle |
| * type’s import operations. [...] If the semaphore was using a |
| * temporarily imported payload, the semaphore’s prior permanent payload |
| * will be restored. |
| */ |
| if (impl == &semaphore->temporary) |
| anv_semaphore_impl_cleanup(device, impl); |
| |
| return VK_SUCCESS; |
| } |
| |
| VkResult anv_GetSemaphoreCounterValue( |
| VkDevice _device, |
| VkSemaphore _semaphore, |
| uint64_t* pValue) |
| { |
| ANV_FROM_HANDLE(anv_device, device, _device); |
| ANV_FROM_HANDLE(anv_semaphore, semaphore, _semaphore); |
| |
| struct anv_semaphore_impl *impl = |
| semaphore->temporary.type != ANV_SEMAPHORE_TYPE_NONE ? |
| &semaphore->temporary : &semaphore->permanent; |
| |
| switch (impl->type) { |
| case ANV_SEMAPHORE_TYPE_TIMELINE: { |
| pthread_mutex_lock(&device->mutex); |
| *pValue = impl->timeline.highest_past; |
| pthread_mutex_unlock(&device->mutex); |
| return VK_SUCCESS; |
| } |
| |
| default: |
| unreachable("Invalid semaphore type"); |
| } |
| } |
| |
| static VkResult |
| anv_timeline_wait_locked(struct anv_device *device, |
| struct anv_timeline *timeline, |
| uint64_t serial, uint64_t abs_timeout_ns) |
| { |
| /* Wait on the queue_submit condition variable until the timeline has a |
| * time point pending that's at least as high as serial. |
| */ |
| while (timeline->highest_pending < serial) { |
| struct timespec abstime = { |
| .tv_sec = abs_timeout_ns / NSEC_PER_SEC, |
| .tv_nsec = abs_timeout_ns % NSEC_PER_SEC, |
| }; |
| |
| int ret = pthread_cond_timedwait(&device->queue_submit, |
| &device->mutex, &abstime); |
| assert(ret != EINVAL); |
| if (anv_gettime_ns() >= abs_timeout_ns && |
| timeline->highest_pending < serial) |
| return VK_TIMEOUT; |
| } |
| |
| while (1) { |
| VkResult result = anv_timeline_gc_locked(device, timeline); |
| if (result != VK_SUCCESS) |
| return result; |
| |
| if (timeline->highest_past >= serial) |
| return VK_SUCCESS; |
| |
| /* If we got here, our earliest time point has a busy BO */ |
| struct anv_timeline_point *point = |
| list_first_entry(&timeline->points, |
| struct anv_timeline_point, link); |
| |
| /* Drop the lock while we wait. */ |
| point->waiting++; |
| pthread_mutex_unlock(&device->mutex); |
| |
| result = anv_device_wait(device, point->bo, |
| anv_get_relative_timeout(abs_timeout_ns)); |
| |
| /* Pick the mutex back up */ |
| pthread_mutex_lock(&device->mutex); |
| point->waiting--; |
| |
| /* This covers both VK_TIMEOUT and VK_ERROR_DEVICE_LOST */ |
| if (result != VK_SUCCESS) |
| return result; |
| } |
| } |
| |
| static VkResult |
| anv_timelines_wait(struct anv_device *device, |
| struct anv_timeline **timelines, |
| const uint64_t *serials, |
| uint32_t n_timelines, |
| bool wait_all, |
| uint64_t abs_timeout_ns) |
| { |
| if (!wait_all && n_timelines > 1) { |
| while (1) { |
| VkResult result; |
| pthread_mutex_lock(&device->mutex); |
| for (uint32_t i = 0; i < n_timelines; i++) { |
| result = |
| anv_timeline_wait_locked(device, timelines[i], serials[i], 0); |
| if (result != VK_TIMEOUT) |
| break; |
| } |
| |
| if (result != VK_TIMEOUT || |
| anv_gettime_ns() >= abs_timeout_ns) { |
| pthread_mutex_unlock(&device->mutex); |
| return result; |
| } |
| |
| /* If none of them are ready do a short wait so we don't completely |
| * spin while holding the lock. The 10us is completely arbitrary. |
| */ |
| uint64_t abs_short_wait_ns = |
| anv_get_absolute_timeout( |
| MIN2((anv_gettime_ns() - abs_timeout_ns) / 10, 10 * 1000)); |
| struct timespec abstime = { |
| .tv_sec = abs_short_wait_ns / NSEC_PER_SEC, |
| .tv_nsec = abs_short_wait_ns % NSEC_PER_SEC, |
| }; |
| ASSERTED int ret; |
| ret = pthread_cond_timedwait(&device->queue_submit, |
| &device->mutex, &abstime); |
| assert(ret != EINVAL); |
| } |
| } else { |
| VkResult result = VK_SUCCESS; |
| pthread_mutex_lock(&device->mutex); |
| for (uint32_t i = 0; i < n_timelines; i++) { |
| result = |
| anv_timeline_wait_locked(device, timelines[i], |
| serials[i], abs_timeout_ns); |
| if (result != VK_SUCCESS) |
| break; |
| } |
| pthread_mutex_unlock(&device->mutex); |
| return result; |
| } |
| } |
| |
| VkResult anv_WaitSemaphores( |
| VkDevice _device, |
| const VkSemaphoreWaitInfoKHR* pWaitInfo, |
| uint64_t timeout) |
| { |
| ANV_FROM_HANDLE(anv_device, device, _device); |
| |
| if (device->no_hw) |
| return VK_SUCCESS; |
| |
| struct anv_timeline **timelines = |
| vk_alloc(&device->vk.alloc, |
| pWaitInfo->semaphoreCount * sizeof(*timelines), |
| 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); |
| if (!timelines) |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| |
| uint64_t *values = vk_alloc(&device->vk.alloc, |
| pWaitInfo->semaphoreCount * sizeof(*values), |
| 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); |
| if (!values) { |
| vk_free(&device->vk.alloc, timelines); |
| return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); |
| } |
| |
| uint32_t handle_count = 0; |
| for (uint32_t i = 0; i < pWaitInfo->semaphoreCount; i++) { |
| ANV_FROM_HANDLE(anv_semaphore, semaphore, pWaitInfo->pSemaphores[i]); |
| struct anv_semaphore_impl *impl = |
| semaphore->temporary.type != ANV_SEMAPHORE_TYPE_NONE ? |
| &semaphore->temporary : &semaphore->permanent; |
| |
| assert(impl->type == ANV_SEMAPHORE_TYPE_TIMELINE); |
| |
| if (pWaitInfo->pValues[i] == 0) |
| continue; |
| |
| timelines[handle_count] = &impl->timeline; |
| values[handle_count] = pWaitInfo->pValues[i]; |
| handle_count++; |
| } |
| |
| VkResult result = VK_SUCCESS; |
| if (handle_count > 0) { |
| result = anv_timelines_wait(device, timelines, values, handle_count, |
| !(pWaitInfo->flags & VK_SEMAPHORE_WAIT_ANY_BIT_KHR), |
| timeout); |
| } |
| |
| vk_free(&device->vk.alloc, timelines); |
| vk_free(&device->vk.alloc, values); |
| |
| return result; |
| } |
| |
| VkResult anv_SignalSemaphore( |
| VkDevice _device, |
| const VkSemaphoreSignalInfoKHR* pSignalInfo) |
| { |
| ANV_FROM_HANDLE(anv_device, device, _device); |
| ANV_FROM_HANDLE(anv_semaphore, semaphore, pSignalInfo->semaphore); |
| |
| struct anv_semaphore_impl *impl = |
| semaphore->temporary.type != ANV_SEMAPHORE_TYPE_NONE ? |
| &semaphore->temporary : &semaphore->permanent; |
| |
| switch (impl->type) { |
| case ANV_SEMAPHORE_TYPE_TIMELINE: { |
| pthread_mutex_lock(&device->mutex); |
| |
| VkResult result = anv_timeline_gc_locked(device, &impl->timeline); |
| |
| assert(pSignalInfo->value > impl->timeline.highest_pending); |
| |
| impl->timeline.highest_pending = impl->timeline.highest_past = pSignalInfo->value; |
| |
| if (result == VK_SUCCESS) |
| result = anv_device_submit_deferred_locked(device); |
| |
| pthread_cond_broadcast(&device->queue_submit); |
| pthread_mutex_unlock(&device->mutex); |
| return result; |
| } |
| |
| default: |
| unreachable("Invalid semaphore type"); |
| } |
| } |