src/amd/vulkan/radv_query.c - platform/external/mesa3d - Git at Google

 /*
  * Copyrigh 2016 Red Hat Inc.
  * Based on anv:
  * Copyright © 2015 Intel Corporation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
  * to deal in the Software without restriction, including without limitation
  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  * and/or sell copies of the Software, and to permit persons to whom the
  * Software is furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice (including the next
  * paragraph) shall be included in all copies or substantial portions of the
  * Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  * IN THE SOFTWARE.
  */

 #include <assert.h>
 #include <stdbool.h>
 #include <string.h>
 #include <unistd.h>
 #include <fcntl.h>

 #include "radv_private.h"
 #include "radv_cs.h"
 #include "sid.h"

 static unsigned get_max_db(struct radv_device *device)
 {
 	unsigned num_db = device->physical_device->rad_info.num_render_backends;
 	MAYBE_UNUSED unsigned rb_mask = device->physical_device->rad_info.enabled_rb_mask;

 	if (device->physical_device->rad_info.chip_class == SI)
 		num_db = 8;
 	else
 		num_db = MAX2(8, num_db);

 	/* Otherwise we need to change the query reset procedure */
 	assert(rb_mask == ((1ull << num_db) - 1));

 	return num_db;
 }

 VkResult radv_CreateQueryPool(
 	VkDevice                                    _device,
 	const VkQueryPoolCreateInfo*                pCreateInfo,
 	const VkAllocationCallbacks*                pAllocator,
 	VkQueryPool*                                pQueryPool)
 {
 	RADV_FROM_HANDLE(radv_device, device, _device);
 	uint64_t size;
 	struct radv_query_pool *pool = vk_alloc2(&device->alloc, pAllocator,
 					       sizeof(*pool), 8,
 					       VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);

 	if (!pool)
 		return VK_ERROR_OUT_OF_HOST_MEMORY;


 	switch(pCreateInfo->queryType) {
 	case VK_QUERY_TYPE_OCCLUSION:
 		/* 16 bytes tmp. buffer as the compute packet writes 64 bits, but
 		 * the app. may have 32 bits of space. */
 		pool->stride = 16 * get_max_db(device) + 16;
 		break;
 	case VK_QUERY_TYPE_PIPELINE_STATISTICS:
 		pool->stride = 16 * 11;
 		break;
 	case VK_QUERY_TYPE_TIMESTAMP:
 		pool->stride = 8;
 		break;
 	default:
 		unreachable("creating unhandled query type");
 	}

 	pool->type = pCreateInfo->queryType;
 	pool->availability_offset = pool->stride * pCreateInfo->queryCount;
 	size = pool->availability_offset + 4 * pCreateInfo->queryCount;

 	pool->bo = device->ws->buffer_create(device->ws, size,
 					     64, RADEON_DOMAIN_GTT, 0);

 	if (!pool->bo) {
 		vk_free2(&device->alloc, pAllocator, pool);
 		return VK_ERROR_OUT_OF_DEVICE_MEMORY;
 	}

 	pool->ptr = device->ws->buffer_map(pool->bo);

 	if (!pool->ptr) {
 		device->ws->buffer_destroy(pool->bo);
 		vk_free2(&device->alloc, pAllocator, pool);
 		return VK_ERROR_OUT_OF_DEVICE_MEMORY;
 	}
 	memset(pool->ptr, 0, size);

 	*pQueryPool = radv_query_pool_to_handle(pool);
 	return VK_SUCCESS;
 }

 void radv_DestroyQueryPool(
 	VkDevice                                    _device,
 	VkQueryPool                                 _pool,
 	const VkAllocationCallbacks*                pAllocator)
 {
 	RADV_FROM_HANDLE(radv_device, device, _device);
 	RADV_FROM_HANDLE(radv_query_pool, pool, _pool);

 	if (!pool)
 		return;

 	device->ws->buffer_destroy(pool->bo);
 	vk_free2(&device->alloc, pAllocator, pool);
 }

 VkResult radv_GetQueryPoolResults(
 	VkDevice                                    _device,
 	VkQueryPool                                 queryPool,
 	uint32_t                                    firstQuery,
 	uint32_t                                    queryCount,
 	size_t                                      dataSize,
 	void*                                       pData,
 	VkDeviceSize                                stride,
 	VkQueryResultFlags                          flags)
 {
 	RADV_FROM_HANDLE(radv_query_pool, pool, queryPool);
 	char *data = pData;
 	VkResult result = VK_SUCCESS;

 	for(unsigned i = 0; i < queryCount; ++i, data += stride) {
 		char *dest = data;
 		unsigned query = firstQuery + i;
 		char *src = pool->ptr + query * pool->stride;
 		uint32_t available;

 		if (flags & VK_QUERY_RESULT_WAIT_BIT) {
 			while(!*(volatile uint32_t*)(pool->ptr + pool->availability_offset + 4 * query))
 				;
 		}

 		if (!*(uint32_t*)(pool->ptr + pool->availability_offset + 4 * query) &&
 		    !(flags & VK_QUERY_RESULT_PARTIAL_BIT)) {
 			if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT)
 				*(uint32_t*)dest = 0;
 			result = VK_NOT_READY;
 			continue;

 		}

 		available = *(uint32_t*)(pool->ptr + pool->availability_offset + 4 * query);
 		switch (pool->type) {
 		case VK_QUERY_TYPE_TIMESTAMP:
 			if (flags & VK_QUERY_RESULT_64_BIT) {
 				*(uint64_t*)dest = *(uint64_t*)src;
 				dest += 8;
 			} else {
 				*(uint32_t*)dest = *(uint32_t*)src;
 				dest += 4;
 			}
 			break;
 		case VK_QUERY_TYPE_OCCLUSION: {
 			uint64_t result = *(uint64_t*)(src + pool->stride - 16);

 			if (flags & VK_QUERY_RESULT_64_BIT) {
 				*(uint64_t*)dest = result;
 				dest += 8;
 			} else {
 				*(uint32_t*)dest = result;
 				dest += 4;
 			}
 			break;
 		default:
 			unreachable("trying to get results of unhandled query type");
 		}
 		}

 		if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
 			*(uint32_t*)dest = available;
 			dest += 4;
 		}
 	}

 	return result;
 }

 void radv_CmdCopyQueryPoolResults(
     VkCommandBuffer                             commandBuffer,
     VkQueryPool                                 queryPool,
     uint32_t                                    firstQuery,
     uint32_t                                    queryCount,
     VkBuffer                                    dstBuffer,
     VkDeviceSize                                dstOffset,
     VkDeviceSize                                stride,
     VkQueryResultFlags                          flags)
 {
 	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 	RADV_FROM_HANDLE(radv_query_pool, pool, queryPool);
 	RADV_FROM_HANDLE(radv_buffer, dst_buffer, dstBuffer);
 	struct radeon_winsys_cs *cs = cmd_buffer->cs;
 	uint64_t va = cmd_buffer->device->ws->buffer_get_va(pool->bo);
 	uint64_t dest_va = cmd_buffer->device->ws->buffer_get_va(dst_buffer->bo);
 	dest_va += dst_buffer->offset + dstOffset;

 	cmd_buffer->device->ws->cs_add_buffer(cmd_buffer->cs, pool->bo, 8);
 	cmd_buffer->device->ws->cs_add_buffer(cmd_buffer->cs, dst_buffer->bo, 8);

 	for(unsigned i = 0; i < queryCount; ++i, dest_va += stride) {
 		unsigned query = firstQuery + i;
 		uint64_t local_src_va = va  + query * pool->stride;
 		unsigned elem_size = (flags & VK_QUERY_RESULT_64_BIT) ? 8 : 4;

 		MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 26);

 		if (flags & VK_QUERY_RESULT_WAIT_BIT) {
 			/* TODO, not sure if there is any case where we won't always be ready yet */
 			uint64_t avail_va = va + pool->availability_offset + 4 * query;


 			/* This waits on the ME. All copies below are done on the ME */
 			radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0));
 			radeon_emit(cs, WAIT_REG_MEM_EQUAL | WAIT_REG_MEM_MEM_SPACE(1));
 			radeon_emit(cs, avail_va);
 			radeon_emit(cs, avail_va >> 32);
 			radeon_emit(cs, 1); /* reference value */
 			radeon_emit(cs, 0xffffffff); /* mask */
 			radeon_emit(cs, 4); /* poll interval */
 		}

 		switch (pool->type) {
 		case VK_QUERY_TYPE_OCCLUSION:
 			local_src_va += pool->stride - 16;

 		case VK_QUERY_TYPE_TIMESTAMP:
 			radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
 			radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_MEM) |
 					COPY_DATA_DST_SEL(COPY_DATA_MEM) |
 					((flags & VK_QUERY_RESULT_64_BIT) ? COPY_DATA_COUNT_SEL : 0));
 			radeon_emit(cs, local_src_va);
 			radeon_emit(cs, local_src_va >> 32);
 			radeon_emit(cs, dest_va);
 			radeon_emit(cs, dest_va >> 32);
 			break;
 		default:
 			unreachable("trying to get results of unhandled query type");
 		}

 		/* The flag could be still changed while the data copy is busy and we
 		 * then might have invalid data, but a ready flag. However, the availability
 		 * writes happen on the ME too, so they should be synchronized. Might need to
 		 * revisit this with multiple queues.
 		 */
 		if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
 			uint64_t avail_va = va + pool->availability_offset + 4 * query;
 			uint64_t avail_dest_va = dest_va;
 			if (pool->type != VK_QUERY_TYPE_PIPELINE_STATISTICS)
 				avail_dest_va += elem_size;
 			else
 				abort();

 			radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
 			radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_MEM) |
 					COPY_DATA_DST_SEL(COPY_DATA_MEM));
 			radeon_emit(cs, avail_va);
 			radeon_emit(cs, avail_va >> 32);
 			radeon_emit(cs, avail_dest_va);
 			radeon_emit(cs, avail_dest_va >> 32);
 		}

 		assert(cs->cdw <= cdw_max);
 	}

 }

 void radv_CmdResetQueryPool(
 	VkCommandBuffer                             commandBuffer,
 	VkQueryPool                                 queryPool,
 	uint32_t                                    firstQuery,
 	uint32_t                                    queryCount)
 {
 	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 	RADV_FROM_HANDLE(radv_query_pool, pool, queryPool);
 	uint64_t va = cmd_buffer->device->ws->buffer_get_va(pool->bo);

 	cmd_buffer->device->ws->cs_add_buffer(cmd_buffer->cs, pool->bo, 8);

 	si_cp_dma_clear_buffer(cmd_buffer, va + firstQuery * pool->stride,
 			       queryCount * pool->stride, 0);
 	si_cp_dma_clear_buffer(cmd_buffer, va + pool->availability_offset + firstQuery * 4,
 			       queryCount * 4, 0);
 }

 void radv_CmdBeginQuery(
     VkCommandBuffer                             commandBuffer,
     VkQueryPool                                 queryPool,
     uint32_t                                    query,
     VkQueryControlFlags                         flags)
 {
 	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 	RADV_FROM_HANDLE(radv_query_pool, pool, queryPool);
 	struct radeon_winsys_cs *cs = cmd_buffer->cs;
 	uint64_t va = cmd_buffer->device->ws->buffer_get_va(pool->bo);
 	va += pool->stride * query;

 	cmd_buffer->device->ws->cs_add_buffer(cs, pool->bo, 8);

 	switch (pool->type) {
 	case VK_QUERY_TYPE_OCCLUSION:
 		radeon_check_space(cmd_buffer->device->ws, cs, 7);

 		++cmd_buffer->state.active_occlusion_queries;
 		if (cmd_buffer->state.active_occlusion_queries == 1)
 			radv_set_db_count_control(cmd_buffer);

 		radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
 		radeon_emit(cs, EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1));
 		radeon_emit(cs, va);
 		radeon_emit(cs, va >> 32);
 		break;
 	default:
 		unreachable("beginning unhandled query type");
 	}
 }


 void radv_CmdEndQuery(
     VkCommandBuffer                             commandBuffer,
     VkQueryPool                                 queryPool,
     uint32_t                                    query)
 {
 	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 	RADV_FROM_HANDLE(radv_query_pool, pool, queryPool);
 	struct radeon_winsys_cs *cs = cmd_buffer->cs;
 	uint64_t va = cmd_buffer->device->ws->buffer_get_va(pool->bo);
 	uint64_t avail_va = va + pool->availability_offset + 4 * query;
 	va += pool->stride * query;

 	cmd_buffer->device->ws->cs_add_buffer(cs, pool->bo, 8);

 	switch (pool->type) {
 	case VK_QUERY_TYPE_OCCLUSION:
 		radeon_check_space(cmd_buffer->device->ws, cs, 14);

 		cmd_buffer->state.active_occlusion_queries--;
 		if (cmd_buffer->state.active_occlusion_queries == 0)
 			radv_set_db_count_control(cmd_buffer);

 		radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
 		radeon_emit(cs, EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1));
 		radeon_emit(cs, va + 8);
 		radeon_emit(cs, (va + 8) >> 32);

 		radeon_emit(cs, PKT3(PKT3_OCCLUSION_QUERY, 3, 0));
 		radeon_emit(cs, va);
 		radeon_emit(cs, va >> 32);
 		radeon_emit(cs, va + pool->stride - 16);
 		radeon_emit(cs, (va + pool->stride - 16) >> 32);

 		break;
 	default:
 		unreachable("ending unhandled query type");
 	}

 	radeon_check_space(cmd_buffer->device->ws, cs, 5);

 	radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0));
 	radeon_emit(cs, S_370_DST_SEL(V_370_MEMORY_SYNC) |
 		    S_370_WR_CONFIRM(1) |
 		    S_370_ENGINE_SEL(V_370_ME));
 	radeon_emit(cs, avail_va);
 	radeon_emit(cs, avail_va >> 32);
 	radeon_emit(cs, 1);
 }

 void radv_CmdWriteTimestamp(
     VkCommandBuffer                             commandBuffer,
     VkPipelineStageFlagBits                     pipelineStage,
     VkQueryPool                                 queryPool,
     uint32_t                                    query)
 {
 	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
 	RADV_FROM_HANDLE(radv_query_pool, pool, queryPool);
 	bool mec = radv_cmd_buffer_uses_mec(cmd_buffer);
 	struct radeon_winsys_cs *cs = cmd_buffer->cs;
 	uint64_t va = cmd_buffer->device->ws->buffer_get_va(pool->bo);
 	uint64_t avail_va = va + pool->availability_offset + 4 * query;
 	uint64_t query_va = va + pool->stride * query;

 	cmd_buffer->device->ws->cs_add_buffer(cs, pool->bo, 5);

 	MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 12);

 	if (mec) {
 		radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, 5, 0));
 		radeon_emit(cs, EVENT_TYPE(V_028A90_BOTTOM_OF_PIPE_TS) | EVENT_INDEX(5));
 		radeon_emit(cs, 3 << 29);
 		radeon_emit(cs, query_va);
 		radeon_emit(cs, query_va >> 32);
 		radeon_emit(cs, 0);
 		radeon_emit(cs, 0);
 	} else {
 		radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, 0));
 		radeon_emit(cs, EVENT_TYPE(V_028A90_BOTTOM_OF_PIPE_TS) | EVENT_INDEX(5));
 		radeon_emit(cs, query_va);
 		radeon_emit(cs, (3 << 29) | ((query_va >> 32) & 0xFFFF));
 		radeon_emit(cs, 0);
 		radeon_emit(cs, 0);
 	}

 	radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0));
 	radeon_emit(cs, S_370_DST_SEL(mec ? V_370_MEM_ASYNC : V_370_MEMORY_SYNC) |
 		    S_370_WR_CONFIRM(1) |
 		    S_370_ENGINE_SEL(V_370_ME));
 	radeon_emit(cs, avail_va);
 	radeon_emit(cs, avail_va >> 32);
 	radeon_emit(cs, 1);

 	assert(cmd_buffer->cs->cdw <= cdw_max);
 }
	/*
	* Copyrigh 2016 Red Hat Inc.
	* Based on anv:
	* Copyright © 2015 Intel Corporation
	*
	* Permission is hereby granted, free of charge, to any person obtaining a
	* copy of this software and associated documentation files (the "Software"),
	* to deal in the Software without restriction, including without limitation
	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	* and/or sell copies of the Software, and to permit persons to whom the
	* Software is furnished to do so, subject to the following conditions:
	*
	* The above copyright notice and this permission notice (including the next
	* paragraph) shall be included in all copies or substantial portions of the
	* Software.
	*
	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
	* IN THE SOFTWARE.
	*/

	#include <assert.h>
	#include <stdbool.h>
	#include <string.h>
	#include <unistd.h>
	#include <fcntl.h>

	#include "radv_private.h"
	#include "radv_cs.h"
	#include "sid.h"

	static unsigned get_max_db(struct radv_device *device)
	{
	unsigned num_db = device->physical_device->rad_info.num_render_backends;
	MAYBE_UNUSED unsigned rb_mask = device->physical_device->rad_info.enabled_rb_mask;

	if (device->physical_device->rad_info.chip_class == SI)
	num_db = 8;
	else
	num_db = MAX2(8, num_db);

	/* Otherwise we need to change the query reset procedure */
	assert(rb_mask == ((1ull << num_db) - 1));

	return num_db;
	}

	VkResult radv_CreateQueryPool(
	VkDevice _device,
	const VkQueryPoolCreateInfo* pCreateInfo,
	const VkAllocationCallbacks* pAllocator,
	VkQueryPool* pQueryPool)
	{
	RADV_FROM_HANDLE(radv_device, device, _device);
	uint64_t size;
	struct radv_query_pool *pool = vk_alloc2(&device->alloc, pAllocator,
	sizeof(*pool), 8,
	VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);

	if (!pool)
	return VK_ERROR_OUT_OF_HOST_MEMORY;


	switch(pCreateInfo->queryType) {
	case VK_QUERY_TYPE_OCCLUSION:
	/* 16 bytes tmp. buffer as the compute packet writes 64 bits, but
	* the app. may have 32 bits of space. */
	pool->stride = 16 * get_max_db(device) + 16;
	break;
	case VK_QUERY_TYPE_PIPELINE_STATISTICS:
	pool->stride = 16 * 11;
	break;
	case VK_QUERY_TYPE_TIMESTAMP:
	pool->stride = 8;
	break;
	default:
	unreachable("creating unhandled query type");
	}

	pool->type = pCreateInfo->queryType;
	pool->availability_offset = pool->stride * pCreateInfo->queryCount;
	size = pool->availability_offset + 4 * pCreateInfo->queryCount;

	pool->bo = device->ws->buffer_create(device->ws, size,
	64, RADEON_DOMAIN_GTT, 0);

	if (!pool->bo) {
	vk_free2(&device->alloc, pAllocator, pool);
	return VK_ERROR_OUT_OF_DEVICE_MEMORY;
	}

	pool->ptr = device->ws->buffer_map(pool->bo);

	if (!pool->ptr) {
	device->ws->buffer_destroy(pool->bo);
	vk_free2(&device->alloc, pAllocator, pool);
	return VK_ERROR_OUT_OF_DEVICE_MEMORY;
	}
	memset(pool->ptr, 0, size);

	*pQueryPool = radv_query_pool_to_handle(pool);
	return VK_SUCCESS;
	}

	void radv_DestroyQueryPool(
	VkDevice _device,
	VkQueryPool _pool,
	const VkAllocationCallbacks* pAllocator)
	{
	RADV_FROM_HANDLE(radv_device, device, _device);
	RADV_FROM_HANDLE(radv_query_pool, pool, _pool);

	if (!pool)
	return;

	device->ws->buffer_destroy(pool->bo);
	vk_free2(&device->alloc, pAllocator, pool);
	}

	VkResult radv_GetQueryPoolResults(
	VkDevice _device,
	VkQueryPool queryPool,
	uint32_t firstQuery,
	uint32_t queryCount,
	size_t dataSize,
	void* pData,
	VkDeviceSize stride,
	VkQueryResultFlags flags)
	{
	RADV_FROM_HANDLE(radv_query_pool, pool, queryPool);
	char *data = pData;
	VkResult result = VK_SUCCESS;

	for(unsigned i = 0; i < queryCount; ++i, data += stride) {
	char *dest = data;
	unsigned query = firstQuery + i;
	char src = pool->ptr + query pool->stride;
	uint32_t available;

	if (flags & VK_QUERY_RESULT_WAIT_BIT) {
	while(!(volatile uint32_t)(pool->ptr + pool->availability_offset + 4 * query))
	;
	}

	if (!(uint32_t)(pool->ptr + pool->availability_offset + 4 * query) &&
	!(flags & VK_QUERY_RESULT_PARTIAL_BIT)) {
	if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT)
	(uint32_t)dest = 0;
	result = VK_NOT_READY;
	continue;

	}

	available = (uint32_t)(pool->ptr + pool->availability_offset + 4 * query);
	switch (pool->type) {
	case VK_QUERY_TYPE_TIMESTAMP:
	if (flags & VK_QUERY_RESULT_64_BIT) {
	(uint64_t)dest = (uint64_t)src;
	dest += 8;
	} else {
	(uint32_t)dest = (uint32_t)src;
	dest += 4;
	}
	break;
	case VK_QUERY_TYPE_OCCLUSION: {
	uint64_t result = (uint64_t)(src + pool->stride - 16);

	if (flags & VK_QUERY_RESULT_64_BIT) {
	(uint64_t)dest = result;
	dest += 8;
	} else {
	(uint32_t)dest = result;
	dest += 4;
	}
	break;
	default:
	unreachable("trying to get results of unhandled query type");
	}
	}

	if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
	(uint32_t)dest = available;
	dest += 4;
	}
	}

	return result;
	}

	void radv_CmdCopyQueryPoolResults(
	VkCommandBuffer commandBuffer,
	VkQueryPool queryPool,
	uint32_t firstQuery,
	uint32_t queryCount,
	VkBuffer dstBuffer,
	VkDeviceSize dstOffset,
	VkDeviceSize stride,
	VkQueryResultFlags flags)
	{
	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
	RADV_FROM_HANDLE(radv_query_pool, pool, queryPool);
	RADV_FROM_HANDLE(radv_buffer, dst_buffer, dstBuffer);
	struct radeon_winsys_cs *cs = cmd_buffer->cs;
	uint64_t va = cmd_buffer->device->ws->buffer_get_va(pool->bo);
	uint64_t dest_va = cmd_buffer->device->ws->buffer_get_va(dst_buffer->bo);
	dest_va += dst_buffer->offset + dstOffset;

	cmd_buffer->device->ws->cs_add_buffer(cmd_buffer->cs, pool->bo, 8);
	cmd_buffer->device->ws->cs_add_buffer(cmd_buffer->cs, dst_buffer->bo, 8);

	for(unsigned i = 0; i < queryCount; ++i, dest_va += stride) {
	unsigned query = firstQuery + i;
	uint64_t local_src_va = va + query * pool->stride;
	unsigned elem_size = (flags & VK_QUERY_RESULT_64_BIT) ? 8 : 4;

	MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 26);

	if (flags & VK_QUERY_RESULT_WAIT_BIT) {
	/* TODO, not sure if there is any case where we won't always be ready yet */
	uint64_t avail_va = va + pool->availability_offset + 4 * query;


	/* This waits on the ME. All copies below are done on the ME */
	radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0));
	radeon_emit(cs, WAIT_REG_MEM_EQUAL \| WAIT_REG_MEM_MEM_SPACE(1));
	radeon_emit(cs, avail_va);
	radeon_emit(cs, avail_va >> 32);
	radeon_emit(cs, 1); /* reference value */
	radeon_emit(cs, 0xffffffff); /* mask */
	radeon_emit(cs, 4); /* poll interval */
	}

	switch (pool->type) {
	case VK_QUERY_TYPE_OCCLUSION:
	local_src_va += pool->stride - 16;

	case VK_QUERY_TYPE_TIMESTAMP:
	radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
	radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_MEM) \|
	COPY_DATA_DST_SEL(COPY_DATA_MEM) \|
	((flags & VK_QUERY_RESULT_64_BIT) ? COPY_DATA_COUNT_SEL : 0));
	radeon_emit(cs, local_src_va);
	radeon_emit(cs, local_src_va >> 32);
	radeon_emit(cs, dest_va);
	radeon_emit(cs, dest_va >> 32);
	break;
	default:
	unreachable("trying to get results of unhandled query type");
	}

	/* The flag could be still changed while the data copy is busy and we
	* then might have invalid data, but a ready flag. However, the availability
	* writes happen on the ME too, so they should be synchronized. Might need to
	* revisit this with multiple queues.
	*/
	if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
	uint64_t avail_va = va + pool->availability_offset + 4 * query;
	uint64_t avail_dest_va = dest_va;
	if (pool->type != VK_QUERY_TYPE_PIPELINE_STATISTICS)
	avail_dest_va += elem_size;
	else
	abort();

	radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
	radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_MEM) \|
	COPY_DATA_DST_SEL(COPY_DATA_MEM));
	radeon_emit(cs, avail_va);
	radeon_emit(cs, avail_va >> 32);
	radeon_emit(cs, avail_dest_va);
	radeon_emit(cs, avail_dest_va >> 32);
	}

	assert(cs->cdw <= cdw_max);
	}

	}

	void radv_CmdResetQueryPool(
	VkCommandBuffer commandBuffer,
	VkQueryPool queryPool,
	uint32_t firstQuery,
	uint32_t queryCount)
	{
	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
	RADV_FROM_HANDLE(radv_query_pool, pool, queryPool);
	uint64_t va = cmd_buffer->device->ws->buffer_get_va(pool->bo);

	cmd_buffer->device->ws->cs_add_buffer(cmd_buffer->cs, pool->bo, 8);

	si_cp_dma_clear_buffer(cmd_buffer, va + firstQuery * pool->stride,
	queryCount * pool->stride, 0);
	si_cp_dma_clear_buffer(cmd_buffer, va + pool->availability_offset + firstQuery * 4,
	queryCount * 4, 0);
	}

	void radv_CmdBeginQuery(
	VkCommandBuffer commandBuffer,
	VkQueryPool queryPool,
	uint32_t query,
	VkQueryControlFlags flags)
	{
	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
	RADV_FROM_HANDLE(radv_query_pool, pool, queryPool);
	struct radeon_winsys_cs *cs = cmd_buffer->cs;
	uint64_t va = cmd_buffer->device->ws->buffer_get_va(pool->bo);
	va += pool->stride * query;

	cmd_buffer->device->ws->cs_add_buffer(cs, pool->bo, 8);

	switch (pool->type) {
	case VK_QUERY_TYPE_OCCLUSION:
	radeon_check_space(cmd_buffer->device->ws, cs, 7);

	++cmd_buffer->state.active_occlusion_queries;
	if (cmd_buffer->state.active_occlusion_queries == 1)
	radv_set_db_count_control(cmd_buffer);

	radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
	radeon_emit(cs, EVENT_TYPE(V_028A90_ZPASS_DONE) \| EVENT_INDEX(1));
	radeon_emit(cs, va);
	radeon_emit(cs, va >> 32);
	break;
	default:
	unreachable("beginning unhandled query type");
	}
	}


	void radv_CmdEndQuery(
	VkCommandBuffer commandBuffer,
	VkQueryPool queryPool,
	uint32_t query)
	{
	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
	RADV_FROM_HANDLE(radv_query_pool, pool, queryPool);
	struct radeon_winsys_cs *cs = cmd_buffer->cs;
	uint64_t va = cmd_buffer->device->ws->buffer_get_va(pool->bo);
	uint64_t avail_va = va + pool->availability_offset + 4 * query;
	va += pool->stride * query;

	cmd_buffer->device->ws->cs_add_buffer(cs, pool->bo, 8);

	switch (pool->type) {
	case VK_QUERY_TYPE_OCCLUSION:
	radeon_check_space(cmd_buffer->device->ws, cs, 14);

	cmd_buffer->state.active_occlusion_queries--;
	if (cmd_buffer->state.active_occlusion_queries == 0)
	radv_set_db_count_control(cmd_buffer);

	radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
	radeon_emit(cs, EVENT_TYPE(V_028A90_ZPASS_DONE) \| EVENT_INDEX(1));
	radeon_emit(cs, va + 8);
	radeon_emit(cs, (va + 8) >> 32);

	radeon_emit(cs, PKT3(PKT3_OCCLUSION_QUERY, 3, 0));
	radeon_emit(cs, va);
	radeon_emit(cs, va >> 32);
	radeon_emit(cs, va + pool->stride - 16);
	radeon_emit(cs, (va + pool->stride - 16) >> 32);

	break;
	default:
	unreachable("ending unhandled query type");
	}

	radeon_check_space(cmd_buffer->device->ws, cs, 5);

	radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0));
	radeon_emit(cs, S_370_DST_SEL(V_370_MEMORY_SYNC) \|
	S_370_WR_CONFIRM(1) \|
	S_370_ENGINE_SEL(V_370_ME));
	radeon_emit(cs, avail_va);
	radeon_emit(cs, avail_va >> 32);
	radeon_emit(cs, 1);
	}

	void radv_CmdWriteTimestamp(
	VkCommandBuffer commandBuffer,
	VkPipelineStageFlagBits pipelineStage,
	VkQueryPool queryPool,
	uint32_t query)
	{
	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
	RADV_FROM_HANDLE(radv_query_pool, pool, queryPool);
	bool mec = radv_cmd_buffer_uses_mec(cmd_buffer);
	struct radeon_winsys_cs *cs = cmd_buffer->cs;
	uint64_t va = cmd_buffer->device->ws->buffer_get_va(pool->bo);
	uint64_t avail_va = va + pool->availability_offset + 4 * query;
	uint64_t query_va = va + pool->stride * query;

	cmd_buffer->device->ws->cs_add_buffer(cs, pool->bo, 5);

	MAYBE_UNUSED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 12);

	if (mec) {
	radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, 5, 0));
	radeon_emit(cs, EVENT_TYPE(V_028A90_BOTTOM_OF_PIPE_TS) \| EVENT_INDEX(5));
	radeon_emit(cs, 3 << 29);
	radeon_emit(cs, query_va);
	radeon_emit(cs, query_va >> 32);
	radeon_emit(cs, 0);
	radeon_emit(cs, 0);
	} else {
	radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, 0));
	radeon_emit(cs, EVENT_TYPE(V_028A90_BOTTOM_OF_PIPE_TS) \| EVENT_INDEX(5));
	radeon_emit(cs, query_va);
	radeon_emit(cs, (3 << 29) \| ((query_va >> 32) & 0xFFFF));
	radeon_emit(cs, 0);
	radeon_emit(cs, 0);
	}

	radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0));
	radeon_emit(cs, S_370_DST_SEL(mec ? V_370_MEM_ASYNC : V_370_MEMORY_SYNC) \|
	S_370_WR_CONFIRM(1) \|
	S_370_ENGINE_SEL(V_370_ME));
	radeon_emit(cs, avail_va);
	radeon_emit(cs, avail_va >> 32);
	radeon_emit(cs, 1);

	assert(cmd_buffer->cs->cdw <= cdw_max);
	}