| // Copyright 2018 The clvk authors. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| #include <array> |
| #include <fstream> |
| #include <functional> |
| #include <iterator> |
| #include <sstream> |
| |
| #include "config.hpp" |
| #include "device.hpp" |
| #include "init.hpp" |
| #include "kernel.hpp" |
| #include "log.hpp" |
| #include "memory.hpp" |
| |
| constexpr VkMemoryPropertyFlags cvk_device::buffer_supported_memory_types[]; |
| constexpr VkMemoryPropertyFlags cvk_device::image_supported_memory_types[]; |
| |
| cvk_device* cvk_device::create(cvk_platform* platform, VkInstance instance, |
| VkPhysicalDevice pdev) { |
| cvk_device* device = new cvk_device(platform, pdev); |
| |
| if (!device->init(instance)) { |
| delete device; |
| return nullptr; |
| } |
| |
| return device; |
| } |
| |
| void cvk_device::init_vulkan_properties(VkInstance instance) { |
| |
| cvk_info("Getting Vulkan device properties"); |
| |
| m_device_id_properties.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES_KHR; |
| m_driver_properties.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES_KHR; |
| m_pci_bus_info_properties.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PCI_BUS_INFO_PROPERTIES_EXT; |
| m_subgroup_properties.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES; |
| m_subgroup_size_control_properties.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_PROPERTIES; |
| m_float_controls_properties.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FLOAT_CONTROLS_PROPERTIES; |
| m_integer_dot_product_properties.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_INTEGER_DOT_PRODUCT_PROPERTIES; |
| |
| //--- Get maxMemoryAllocationSize for figuring out the max single buffer |
| // allocation size and default init when the extension is not supported |
| m_maintenance3_properties.maxMemoryAllocationSize = |
| std::numeric_limits<VkDeviceSize>::max(); |
| m_maintenance3_properties.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_3_PROPERTIES; |
| //--- |
| |
| #define VER_EXT_PROP(ver, ext, prop) \ |
| {ver, ext, reinterpret_cast<VkBaseOutStructure*>(&prop)} |
| std::vector<std::tuple<uint32_t, const char*, VkBaseOutStructure*>> |
| coreversion_extension_properties = { |
| VER_EXT_PROP(VK_MAKE_VERSION(1, 1, 0), nullptr, |
| m_device_id_properties), |
| VER_EXT_PROP(VK_MAKE_VERSION(1, 2, 0), |
| VK_KHR_DRIVER_PROPERTIES_EXTENSION_NAME, |
| m_driver_properties), |
| VER_EXT_PROP(0, VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, |
| m_pci_bus_info_properties), |
| VER_EXT_PROP(VK_MAKE_VERSION(1, 1, 0), nullptr, |
| m_subgroup_properties), |
| VER_EXT_PROP(VK_MAKE_VERSION(1, 3, 0), nullptr, |
| m_subgroup_size_control_properties), |
| VER_EXT_PROP(VK_MAKE_VERSION(1, 1, 0), nullptr, |
| m_maintenance3_properties), |
| VER_EXT_PROP(VK_MAKE_VERSION(1, 2, 0), nullptr, |
| m_float_controls_properties), |
| VER_EXT_PROP(VK_MAKE_VERSION(1, 3, 0), nullptr, |
| m_integer_dot_product_properties), |
| }; |
| #undef VER_EXT_PROP |
| |
| VkBaseOutStructure* pNext = nullptr; |
| for (auto& ver_ext_prop : coreversion_extension_properties) { |
| auto corever = std::get<0>(ver_ext_prop); |
| auto ext = std::get<1>(ver_ext_prop); |
| auto prop = std::get<2>(ver_ext_prop); |
| if ((corever != 0) && (m_properties.apiVersion >= corever)) { |
| prop->pNext = pNext; |
| pNext = prop; |
| } else if ((ext != nullptr) && is_vulkan_extension_enabled(ext)) { |
| prop->pNext = pNext; |
| pNext = prop; |
| } |
| } |
| |
| VkPhysicalDeviceProperties2KHR properties; |
| properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2_KHR; |
| properties.pNext = pNext; |
| |
| auto func = GET_INSTANCE_PROC(instance, vkGetPhysicalDeviceProperties2KHR); |
| if (!func) { |
| cvk_fatal( |
| "Failed to get pointer to vkGetPhysicalDeviceProperties2KHR()"); |
| } |
| func(m_pdev, &properties); |
| } |
| |
| void cvk_device::init_clvk_runtime_behaviors() { |
| m_clvk_properties = create_cvk_device_properties( |
| m_properties.deviceName, m_properties.vendorID, m_properties.deviceID, |
| m_properties.driverVersion, m_driver_properties.driverID); |
| #define SET_DEVICE_PROPERTY(option, print) \ |
| do { \ |
| if (config.option.set) { \ |
| m_##option = config.option; \ |
| } else { \ |
| m_##option = m_clvk_properties->get_##option(); \ |
| } \ |
| print(option); \ |
| } while (0) |
| |
| #define PRINT_U(option) cvk_info_fn(#option ": %u", m_##option); |
| #define SET_DEVICE_PROPERTY_U(option) SET_DEVICE_PROPERTY(option, PRINT_U) |
| |
| #define PRINT_S(option) cvk_info_fn(#option ": %s", m_##option.c_str()); |
| #define SET_DEVICE_PROPERTY_S(option) SET_DEVICE_PROPERTY(option, PRINT_S) |
| |
| SET_DEVICE_PROPERTY_U(max_cmd_batch_size); |
| SET_DEVICE_PROPERTY_U(max_first_cmd_batch_size); |
| SET_DEVICE_PROPERTY_U(max_cmd_group_size); |
| SET_DEVICE_PROPERTY_U(max_first_cmd_group_size); |
| |
| SET_DEVICE_PROPERTY_U(physical_addressing); |
| SET_DEVICE_PROPERTY_S(spirv_arch); |
| |
| SET_DEVICE_PROPERTY_U(preferred_subgroup_size); |
| |
| #undef PRINT_U |
| #undef PRINT_S |
| #undef SET_DEVICE_PROPERTY_U |
| #undef SET_DEVICE_PROPERTY_S |
| #undef SET_DEVICE_PROPERTY |
| } |
| |
| void cvk_device::init_driver_behaviors() { |
| |
| cvk_info("Initialising driver behaviors"); |
| |
| // Disable all driver-specific behaviors by default |
| m_driver_behaviors = 0; |
| |
| if (is_vulkan_extension_enabled(VK_KHR_DRIVER_PROPERTIES_EXTENSION_NAME) || |
| m_properties.apiVersion >= VK_MAKE_VERSION(1, 2, 0)) { |
| |
| // Log basic information about the target Vulkan device |
| cvk_info(" driverName = %s", m_driver_properties.driverName); |
| cvk_info(" driverInfo = %s", m_driver_properties.driverInfo); |
| cvk_info(" conformanceVersion = %d.%d.%d.%d", |
| m_driver_properties.conformanceVersion.major, |
| m_driver_properties.conformanceVersion.minor, |
| m_driver_properties.conformanceVersion.subminor, |
| m_driver_properties.conformanceVersion.patch); |
| |
| // Select behaviors based on the target Vulkan device and driver version |
| if (m_driver_properties.driverID == VK_DRIVER_ID_ARM_PROPRIETARY_KHR) { |
| // Workaround for resource management bug on Mali GPUs. |
| // TODO: Make this conditional on the driver version when this is |
| // fixed in the driver. |
| m_driver_behaviors |= use_reset_command_buffer_bit; |
| } |
| } else { |
| cvk_warn("The VK_KHR_driver_properties extension is not supported."); |
| cvk_warn("Using default Vulkan driver behaviors."); |
| } |
| |
| // List driver behaviors |
| cvk_info("Driver behaviors:"); |
| #define PRINT_BEHAVIOR(name) \ |
| cvk_info(" %s = %s", #name, (m_driver_behaviors & name) ? "true" : "false") |
| PRINT_BEHAVIOR(use_reset_command_buffer_bit); |
| #undef PRINT_BEHAVIOR |
| } |
| |
| static bool queue_flags_contains_compute(VkQueueFlags flags) { |
| return flags & VK_QUEUE_COMPUTE_BIT; |
| } |
| static bool queue_flags_contains_compute_not_graphics(VkQueueFlags flags) { |
| if (flags & VK_QUEUE_GRAPHICS_BIT) { |
| return false; |
| } |
| return queue_flags_contains_compute(flags); |
| } |
| |
| bool cvk_device::init_queues(uint32_t* num_queues, uint32_t* queue_family) { |
| // Get number of queue families |
| uint32_t num_families; |
| vkGetPhysicalDeviceQueueFamilyProperties(m_pdev, &num_families, nullptr); |
| |
| cvk_info( |
| "Physical device (%s) has %u queue families:", |
| vulkan_physical_device_type_string(m_properties.deviceType).c_str(), |
| num_families); |
| |
| // Get their properties |
| std::vector<VkQueueFamilyProperties> families(num_families); |
| vkGetPhysicalDeviceQueueFamilyProperties(m_pdev, &num_families, |
| families.data()); |
| |
| std::array<std::function<bool(VkQueueFlags)>, 2> queue_flags_tests = { |
| queue_flags_contains_compute_not_graphics, |
| queue_flags_contains_compute}; |
| |
| // Look for suitable queues |
| bool found_queues = false; |
| *num_queues = 0; |
| for (auto queue_flags_test : queue_flags_tests) { |
| for (uint32_t i = 0; i < num_families; i++) { |
| |
| cvk_info(" queue family %u: %2u queues | %s", i, |
| families[i].queueCount, |
| vulkan_queue_flags_string(families[i].queueFlags).c_str()); |
| |
| if (!found_queues && queue_flags_test(families[i].queueFlags)) { |
| *queue_family = i; |
| *num_queues = families[i].queueCount; |
| found_queues = true; |
| } |
| } |
| } |
| |
| if (!found_queues) { |
| cvk_error("Could not find a suitable queue family for this device"); |
| return false; |
| } |
| cvk_info( |
| " selecting queue %u: %2u queues | %s", *queue_family, *num_queues, |
| vulkan_queue_flags_string(families[*queue_family].queueFlags).c_str()); |
| |
| // Initialise the queue allocator |
| m_vulkan_queue_alloc_index = 0; |
| |
| return true; |
| } |
| |
| bool cvk_device::init_extensions() { |
| uint32_t numext; |
| VkResult res = |
| vkEnumerateDeviceExtensionProperties(m_pdev, nullptr, &numext, nullptr); |
| CVK_VK_CHECK_ERROR_RET( |
| res, false, "Failed to get the number of device extension properties"); |
| |
| cvk_info("%u extensions are supported", numext); |
| |
| std::vector<VkExtensionProperties> extensions(numext); |
| res = vkEnumerateDeviceExtensionProperties(m_pdev, nullptr, &numext, |
| extensions.data()); |
| CVK_VK_CHECK_ERROR_RET(res, false, |
| "Could not enumerate device extension properties"); |
| |
| std::vector<const char*> desired_extensions{ |
| VK_KHR_8BIT_STORAGE_EXTENSION_NAME, |
| VK_KHR_16BIT_STORAGE_EXTENSION_NAME, |
| VK_KHR_DRIVER_PROPERTIES_EXTENSION_NAME, |
| VK_KHR_SHADER_FLOAT16_INT8_EXTENSION_NAME, |
| VK_KHR_SHADER_NON_SEMANTIC_INFO_EXTENSION_NAME, |
| VK_KHR_UNIFORM_BUFFER_STANDARD_LAYOUT_EXTENSION_NAME, |
| VK_KHR_VARIABLE_POINTERS_EXTENSION_NAME, |
| VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME, |
| VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, |
| VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME, |
| VK_KHR_VULKAN_MEMORY_MODEL_EXTENSION_NAME, |
| VK_EXT_DESCRIPTOR_INDEXING_EXTENSION_NAME, |
| VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME, |
| VK_KHR_GLOBAL_PRIORITY_EXTENSION_NAME, |
| }; |
| |
| if (m_properties.apiVersion < VK_MAKE_VERSION(1, 2, 0)) { |
| desired_extensions.push_back(VK_KHR_SPIRV_1_4_EXTENSION_NAME); |
| desired_extensions.push_back( |
| VK_KHR_SHADER_SUBGROUP_EXTENDED_TYPES_EXTENSION_NAME); |
| } |
| |
| if (m_properties.apiVersion < VK_MAKE_VERSION(1, 1, 0)) { |
| desired_extensions.push_back(VK_KHR_MAINTENANCE3_EXTENSION_NAME); |
| } |
| |
| if (m_properties.apiVersion < VK_MAKE_VERSION(1, 1, 0)) { |
| m_vulkan_device_extensions.push_back( |
| VK_KHR_STORAGE_BUFFER_STORAGE_CLASS_EXTENSION_NAME); |
| } else { |
| desired_extensions.push_back( |
| VK_KHR_STORAGE_BUFFER_STORAGE_CLASS_EXTENSION_NAME); |
| } |
| |
| for (size_t i = 0; i < numext; i++) { |
| cvk_info(" %s, spec version %u", extensions[i].extensionName, |
| extensions[i].specVersion); |
| |
| for (auto name : desired_extensions) { |
| if (!strcmp(name, extensions[i].extensionName)) { |
| m_vulkan_device_extensions.push_back(name); |
| cvk_info(" ENABLING"); |
| break; |
| } |
| } |
| } |
| |
| return true; |
| } |
| |
| void cvk_device::init_features(VkInstance instance) { |
| |
| cvk_info("Initialising features"); |
| |
| // Query supported features. |
| m_features_ubo_stdlayout.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_UNIFORM_BUFFER_STANDARD_LAYOUT_FEATURES_KHR; |
| m_features_float16_int8.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT16_INT8_FEATURES_KHR; |
| m_features_variable_pointer.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VARIABLE_POINTERS_FEATURES_KHR; |
| m_features_8bit_storage.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_8BIT_STORAGE_FEATURES_KHR; |
| m_features_16bit_storage.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_16BIT_STORAGE_FEATURES_KHR; |
| m_features_shader_subgroup_extended_types.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_SUBGROUP_EXTENDED_TYPES_FEATURES_KHR; |
| m_features_vulkan_memory_model.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_MEMORY_MODEL_FEATURES; |
| m_features_buffer_device_address.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_DEVICE_ADDRESS_FEATURES_KHR; |
| m_features_subgroup_size_control.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES; |
| m_features_shader_integer_dot_product.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_INTEGER_DOT_PRODUCT_FEATURES; |
| m_features_queue_global_priority.sType = |
| VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GLOBAL_PRIORITY_QUERY_FEATURES_KHR; |
| |
| std::vector<std::tuple<uint32_t, const char*, VkBaseOutStructure*>> |
| coreversion_extension_features = { |
| #define VER_EXT_FEAT(ver, ext, feat) \ |
| {ver, ext, reinterpret_cast<VkBaseOutStructure*>(&feat)} |
| VER_EXT_FEAT(VK_MAKE_VERSION(1, 2, 0), |
| VK_KHR_UNIFORM_BUFFER_STANDARD_LAYOUT_EXTENSION_NAME, |
| m_features_ubo_stdlayout), |
| VER_EXT_FEAT(VK_MAKE_VERSION(1, 2, 0), |
| VK_KHR_SHADER_FLOAT16_INT8_EXTENSION_NAME, |
| m_features_float16_int8), |
| VER_EXT_FEAT(VK_MAKE_VERSION(1, 1, 0), |
| VK_KHR_VARIABLE_POINTERS_EXTENSION_NAME, |
| m_features_variable_pointer), |
| VER_EXT_FEAT(VK_MAKE_VERSION(1, 2, 0), |
| VK_KHR_8BIT_STORAGE_EXTENSION_NAME, |
| m_features_8bit_storage), |
| VER_EXT_FEAT(VK_MAKE_VERSION(1, 1, 0), |
| VK_KHR_16BIT_STORAGE_EXTENSION_NAME, |
| m_features_16bit_storage), |
| VER_EXT_FEAT(VK_MAKE_VERSION(1, 2, 0), |
| VK_KHR_SHADER_SUBGROUP_EXTENDED_TYPES_EXTENSION_NAME, |
| m_features_shader_subgroup_extended_types), |
| VER_EXT_FEAT(VK_MAKE_VERSION(1, 3, 0), |
| VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME, |
| m_features_subgroup_size_control), |
| VER_EXT_FEAT(VK_MAKE_VERSION(1, 2, 0), |
| VK_KHR_VULKAN_MEMORY_MODEL_EXTENSION_NAME, |
| m_features_vulkan_memory_model), |
| VER_EXT_FEAT(VK_MAKE_VERSION(1, 2, 0), |
| VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME, |
| m_features_buffer_device_address), |
| VER_EXT_FEAT(VK_MAKE_VERSION(1, 3, 0), nullptr, |
| m_features_shader_integer_dot_product), |
| VER_EXT_FEAT(0, VK_KHR_GLOBAL_PRIORITY_EXTENSION_NAME, |
| m_features_queue_global_priority), |
| |
| #undef VER_EXT_FEAT |
| }; |
| |
| VkBaseOutStructure* pNext = nullptr; |
| for (auto& ver_ext_feat : coreversion_extension_features) { |
| auto corever = std::get<0>(ver_ext_feat); |
| auto ext = std::get<1>(ver_ext_feat); |
| auto feat = std::get<2>(ver_ext_feat); |
| if ((corever != 0) && (m_properties.apiVersion >= corever)) { |
| feat->pNext = pNext; |
| pNext = feat; |
| } else if ((ext != nullptr) && is_vulkan_extension_enabled(ext)) { |
| feat->pNext = pNext; |
| pNext = feat; |
| } |
| } |
| |
| VkPhysicalDeviceFeatures2 supported_features; |
| supported_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; |
| supported_features.pNext = pNext; |
| |
| if (m_properties.apiVersion < VK_MAKE_VERSION(1, 1, 0)) { |
| // Use the extension on Vulkan 1.0 platforms |
| auto func = reinterpret_cast<PFN_vkGetPhysicalDeviceFeatures2KHR>( |
| vkGetInstanceProcAddr(instance, "vkGetPhysicalDeviceFeatures2KHR")); |
| if (!func) { |
| cvk_fatal( |
| "Failed to get pointer to vkGetPhysicalDeviceFeatures2KHR()"); |
| } |
| func(m_pdev, &supported_features); |
| } else { |
| vkGetPhysicalDeviceFeatures2(m_pdev, &supported_features); |
| } |
| |
| // Log supported features |
| cvk_info("8-bit storage: SSBO = %d, UBO = %d, Push constants = %d", |
| m_features_8bit_storage.storageBuffer8BitAccess, |
| m_features_8bit_storage.uniformAndStorageBuffer8BitAccess, |
| m_features_8bit_storage.storagePushConstant8); |
| cvk_info("16-bit storage: SSBO = %d, UBO = %d, Push constants = %d", |
| m_features_16bit_storage.storageBuffer16BitAccess, |
| m_features_16bit_storage.uniformAndStorageBuffer16BitAccess, |
| m_features_16bit_storage.storagePushConstant16); |
| cvk_info( |
| "subgroup extended types: %d", |
| m_features_shader_subgroup_extended_types.shaderSubgroupExtendedTypes); |
| |
| // Selectively enable core features. |
| if (supported_features.features.shaderInt16) { |
| m_features.features.shaderInt16 = VK_TRUE; |
| } |
| if (supported_features.features.shaderInt64) { |
| m_features.features.shaderInt64 = VK_TRUE; |
| } |
| if (supported_features.features.shaderFloat64) { |
| m_features.features.shaderFloat64 = VK_TRUE; |
| } |
| if (supported_features.features.shaderStorageImageReadWithoutFormat) { |
| m_features.features.shaderStorageImageReadWithoutFormat = VK_TRUE; |
| } |
| if (supported_features.features.shaderStorageImageWriteWithoutFormat) { |
| m_features.features.shaderStorageImageWriteWithoutFormat = VK_TRUE; |
| } |
| |
| // All queried extended features are enabled when supported. |
| m_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; |
| m_features.pNext = pNext; |
| } |
| |
| void cvk_device::init_command_pointers(VkInstance instance) { |
| // Buffer device address |
| if (m_properties.apiVersion >= VK_MAKE_VERSION(1, 2, 0)) { |
| m_vkfns.vkGetBufferDeviceAddressKHR = |
| GET_INSTANCE_PROC(instance, vkGetBufferDeviceAddress); |
| } else if (is_vulkan_extension_enabled( |
| VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME)) { |
| m_vkfns.vkGetBufferDeviceAddressKHR = |
| GET_INSTANCE_PROC(instance, vkGetBufferDeviceAddressKHR); |
| } |
| } |
| |
| void cvk_device::init_compiler_options() { |
| m_device_compiler_options = ""; |
| |
| if (!devices_support_images()) { |
| m_device_compiler_options += " -images=0 "; |
| } |
| |
| // 8-bit storage capability restrictions. |
| if (device_8bit_storage_features().storageBuffer8BitAccess == VK_FALSE) { |
| m_device_compiler_options += " -no-8bit-storage=ssbo "; |
| } |
| if (device_8bit_storage_features().uniformAndStorageBuffer8BitAccess == |
| VK_FALSE) { |
| m_device_compiler_options += " -no-8bit-storage=ubo "; |
| } |
| if (device_8bit_storage_features().storagePushConstant8 == VK_FALSE) { |
| m_device_compiler_options += " -no-8bit-storage=pushconstant "; |
| } |
| |
| // 16-bit storage capability restrictions. |
| if (device_16bit_storage_features().storageBuffer16BitAccess == VK_FALSE) { |
| m_device_compiler_options += " -no-16bit-storage=ssbo "; |
| } |
| if (device_16bit_storage_features().uniformAndStorageBuffer16BitAccess == |
| VK_FALSE) { |
| m_device_compiler_options += " -no-16bit-storage=ubo "; |
| } |
| if (device_16bit_storage_features().storagePushConstant16 == VK_FALSE) { |
| m_device_compiler_options += " -no-16bit-storage=pushconstant "; |
| } |
| std::vector<std::string> roundingModeRTE; |
| if (m_float_controls_properties.shaderRoundingModeRTEFloat16 && |
| supports_fp16()) { |
| roundingModeRTE.push_back("16"); |
| } |
| if (m_float_controls_properties.shaderRoundingModeRTEFloat32) { |
| roundingModeRTE.push_back("32"); |
| } |
| if (m_float_controls_properties.shaderRoundingModeRTEFloat64 && |
| supports_fp64()) { |
| roundingModeRTE.push_back("64"); |
| } |
| if (roundingModeRTE.size() > 0) { |
| m_device_compiler_options += " -rounding-mode-rte="; |
| for (unsigned i = 0; i < roundingModeRTE.size(); i++) { |
| if (i != 0) { |
| m_device_compiler_options += ","; |
| } |
| m_device_compiler_options += roundingModeRTE[i]; |
| } |
| m_device_compiler_options += " "; |
| } |
| |
| // Types support |
| if (!supports_fp16()) { |
| m_device_compiler_options += " -fp16=0 "; |
| } |
| if (!supports_fp64()) { |
| m_device_compiler_options += " -fp64=0 "; |
| } |
| if (supports_int8()) { |
| m_device_compiler_options += " -rewrite-packed-structs "; |
| } else { |
| m_device_compiler_options += " -int8=0 "; |
| } |
| if (supports_ubo_stdlayout()) { |
| m_device_compiler_options += " -std430-ubo-layout "; |
| } |
| if (supports_non_uniform_decoration()) { |
| m_device_compiler_options += " -decorate-nonuniform "; |
| } |
| |
| // Device specific options |
| m_device_compiler_options += |
| " " + m_clvk_properties->get_compile_options() + " "; |
| |
| m_device_compiler_options += " -arch=" + m_spirv_arch + " "; |
| |
| if (m_physical_addressing) { |
| m_device_compiler_options += " -physical-storage-buffers "; |
| } |
| |
| if (supports_dot_product()) { |
| m_device_compiler_options += " -cl-arm-integer-dot-product "; |
| } |
| |
| // Builtin options |
| auto parse_builtins = [](std::string s) { |
| std::set<std::string> builtins; |
| size_t pos = 0; |
| size_t comma = s.find(',', pos); |
| while (comma != std::string::npos) { |
| builtins.insert(s.substr(pos, comma - pos)); |
| pos = comma + 1; |
| comma = s.find(',', pos); |
| } |
| builtins.insert(s.substr(pos)); |
| return builtins; |
| }; |
| auto clspv_library_builtins = |
| parse_builtins(config.clspv_library_builtins()); |
| auto native_builtins = m_clvk_properties->get_native_builtins(); |
| auto clspv_native_builtins = parse_builtins(config.clspv_native_builtins()); |
| native_builtins.insert(clspv_native_builtins.begin(), |
| clspv_native_builtins.end()); |
| |
| std::string builtin_list = ""; |
| for (const auto& builtin : native_builtins) { |
| if (clspv_library_builtins.count(builtin) > 0) { |
| continue; |
| } |
| builtin_list += builtin + ","; |
| } |
| if (builtin_list != "") { |
| m_device_compiler_options += |
| " --use-native-builtins=" + builtin_list + " "; |
| } |
| |
| // Select target SPIR-V version |
| m_device_compiler_options += " -spv-version="; |
| switch (vulkan_spirv_env()) { |
| default: |
| case SPV_ENV_VULKAN_1_0: |
| m_device_compiler_options += "1.0 "; |
| break; |
| case SPV_ENV_VULKAN_1_1: |
| m_device_compiler_options += "1.3 "; |
| break; |
| case SPV_ENV_VULKAN_1_1_SPIRV_1_4: |
| m_device_compiler_options += "1.4 "; |
| break; |
| case SPV_ENV_VULKAN_1_2: |
| m_device_compiler_options += "1.5 "; |
| break; |
| case SPV_ENV_VULKAN_1_3: |
| m_device_compiler_options += "1.6 "; |
| break; |
| } |
| |
| // Limits |
| m_device_compiler_options += |
| " -max-pushconstant-size=" + |
| std::to_string(vulkan_max_push_constants_size()) + " "; |
| m_device_compiler_options += |
| " -max-ubo-size=" + std::to_string(vulkan_max_uniform_buffer_range()) + |
| " "; |
| m_device_compiler_options += " -global-offset "; |
| m_device_compiler_options += " -long-vector "; |
| m_device_compiler_options += " -module-constants-in-storage-buffer "; |
| m_device_compiler_options += " -cl-arm-non-uniform-work-group-size "; |
| } |
| |
| void cvk_device::build_extension_ils_list() { |
| |
| m_extensions = { |
| // Start with required extensions |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_global_int32_base_atomics"), |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_global_int32_extended_atomics"), |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_local_int32_base_atomics"), |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_local_int32_extended_atomics"), |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_byte_addressable_store"), |
| |
| // Add always supported extensions |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_extended_versioning"), |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_create_command_queue"), |
| #ifdef ENABLE_SPIRV_IL |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_il_program"), |
| #endif |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_spirv_no_integer_wrap_decoration"), |
| MAKE_NAME_VERSION(1, 0, 0, "cl_arm_non_uniform_work_group_size"), |
| MAKE_NAME_VERSION(1, 0, 0, "cl_arm_printf"), |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_suggested_local_work_size"), |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_3d_image_writes"), |
| // MAKE_NAME_VERSION(0, 9, 0, "cl_khr_semaphore"), |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_spirv_linkonce_odr"), |
| }; |
| |
| if (m_properties.apiVersion >= VK_MAKE_VERSION(1, 1, 0)) { |
| m_extensions.push_back( |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_device_uuid")); |
| VkSubgroupFeatureFlags required_subgroup_ops = |
| VK_SUBGROUP_FEATURE_BASIC_BIT | VK_SUBGROUP_FEATURE_ARITHMETIC_BIT; |
| if ((m_subgroup_properties.supportedOperations & |
| required_subgroup_ops) == required_subgroup_ops && |
| (m_features_shader_subgroup_extended_types |
| .shaderSubgroupExtendedTypes == VK_TRUE)) { |
| m_has_subgroups_support = true; |
| } |
| if (m_subgroup_properties.supportedOperations & |
| VK_SUBGROUP_FEATURE_SHUFFLE_BIT) { |
| m_extensions.push_back( |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_subgroup_shuffle")); |
| } |
| if (supports_subgroup_rotate()) { |
| m_extensions.push_back( |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_subgroup_rotate")); |
| } |
| if (m_subgroup_properties.supportedOperations & |
| VK_SUBGROUP_FEATURE_BALLOT_BIT) { |
| m_extensions.push_back( |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_subgroup_ballot")); |
| } |
| if (m_subgroup_properties.supportedOperations & |
| VK_SUBGROUP_FEATURE_VOTE_BIT) { |
| m_extensions.push_back( |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_subgroup_non_uniform_vote")); |
| } |
| } |
| |
| // Enable cl_khr_fp16 if we have 16-bit storage and shaderFloat16 |
| if ((is_vulkan_extension_enabled(VK_KHR_16BIT_STORAGE_EXTENSION_NAME) && |
| m_features_16bit_storage.storageBuffer16BitAccess) && |
| (is_vulkan_extension_enabled( |
| VK_KHR_SHADER_FLOAT16_INT8_EXTENSION_NAME) && |
| m_features_float16_int8.shaderFloat16)) { |
| m_has_fp16_support = true; |
| m_extensions.push_back(MAKE_NAME_VERSION(1, 0, 0, "cl_khr_fp16")); |
| } |
| |
| if (supports_fp64()) { |
| m_extensions.push_back(MAKE_NAME_VERSION(1, 0, 0, "cl_khr_fp64")); |
| } |
| |
| // Enable 8-bit integer support if possible |
| if ((is_vulkan_extension_enabled(VK_KHR_8BIT_STORAGE_EXTENSION_NAME) && |
| m_features_8bit_storage.storageBuffer8BitAccess) && |
| (is_vulkan_extension_enabled( |
| VK_KHR_SHADER_FLOAT16_INT8_EXTENSION_NAME) && |
| m_features_float16_int8.shaderInt8)) { |
| m_has_int8_support = true; |
| } |
| |
| // Report cl_khr_pci_bus_info if VK_EXT_pci_bus_info is supported |
| if (is_vulkan_extension_enabled(VK_EXT_PCI_BUS_INFO_EXTENSION_NAME)) { |
| m_extensions.push_back( |
| MAKE_NAME_VERSION(1, 0, 0, "cl_khr_pci_bus_info")); |
| } |
| |
| if ((m_properties.apiVersion >= VK_MAKE_VERSION(1, 3, 0) || |
| is_vulkan_extension_enabled( |
| VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME)) && |
| m_features_subgroup_size_control.subgroupSizeControl && |
| (m_subgroup_size_control_properties.requiredSubgroupSizeStages & |
| VK_SHADER_STAGE_COMPUTE_BIT)) { |
| m_extensions.push_back( |
| MAKE_NAME_VERSION(1, 0, 0, "cl_intel_required_subgroup_size")); |
| m_has_subgroup_size_selection = true; |
| } |
| |
| if (supports_dot_product()) { |
| if (supports_int8()) { |
| m_extensions.push_back(MAKE_NAME_VERSION( |
| 2, 0, 0, CL_KHR_INTEGER_DOT_PRODUCT_EXTENSION_NAME)); |
| m_extensions.push_back( |
| MAKE_NAME_VERSION(1, 0, 0, "cl_arm_integer_dot_product_int8")); |
| m_extensions.push_back(MAKE_NAME_VERSION( |
| 1, 0, 0, "cl_arm_integer_dot_product_accumulate_int8")); |
| m_extensions.push_back(MAKE_NAME_VERSION( |
| 1, 0, 0, |
| "cl_arm_integer_dot_product_accumulate_saturate_int8")); |
| } |
| m_extensions.push_back(MAKE_NAME_VERSION( |
| 1, 0, 0, "cl_arm_integer_dot_product_accumulate_int16")); |
| } |
| |
| auto split_string = [](std::string input, char delimiter) { |
| std::vector<std::string> outputs; |
| size_t pos = 0; |
| while ((pos = input.find(delimiter)) != std::string::npos) { |
| outputs.push_back(input.substr(0, pos)); |
| input.erase(0, pos + 1); |
| } |
| if (input.size() > 0) { |
| outputs.push_back(input); |
| } |
| return outputs; |
| }; |
| auto config_extensions = split_string(config.device_extensions(), ','); |
| for (auto& config_extension : config_extensions) { |
| cl_name_version extension; |
| extension.version = CL_MAKE_VERSION(0, 0, 0); |
| memcpy(extension.name, config_extension.c_str(), |
| std::min(config_extension.size(), |
| (size_t)CL_NAME_VERSION_MAX_NAME_SIZE)); |
| m_extensions.push_back(extension); |
| } |
| |
| auto config_extensions_masked = |
| split_string(config.device_extensions_masked(), ','); |
| for (auto& config_extension_masked : config_extensions_masked) { |
| for (auto it = m_extensions.begin(); it != m_extensions.end(); it++) { |
| if (strcmp(config_extension_masked.c_str(), |
| CL_KHR_FP16_EXTENSION_NAME) == 0) { |
| m_has_fp16_support = false; |
| } |
| if (strcmp(config_extension_masked.c_str(), it->name) == 0) { |
| m_extensions.erase(it); |
| break; |
| } |
| } |
| } |
| |
| // Build extension string |
| for (auto& ext : m_extensions) { |
| m_extension_string += ext.name; |
| m_extension_string += " "; |
| } |
| cvk_info_fn("extensions: '%s'", m_extension_string.c_str()); |
| |
| // Build list of ILs |
| m_ils = { |
| #ifdef ENABLE_SPIRV_IL |
| MAKE_NAME_VERSION(1, 0, 0, "SPIR-V"), |
| #endif |
| }; |
| |
| for (auto& il : m_ils) { |
| m_ils_string += il.name; |
| m_ils_string += "_"; |
| m_ils_string += std::to_string(CL_VERSION_MAJOR(il.version)); |
| m_ils_string += "."; |
| m_ils_string += std::to_string(CL_VERSION_MINOR(il.version)); |
| m_ils_string += " "; |
| } |
| |
| // Build list of supported OpenCL C versions |
| m_opencl_c_versions = { |
| MAKE_NAME_VERSION(1, 0, 0, "OpenCL C"), |
| MAKE_NAME_VERSION(1, 1, 0, "OpenCL C"), |
| MAKE_NAME_VERSION(1, 2, 0, "OpenCL C"), |
| MAKE_NAME_VERSION(3, 0, 0, "OpenCL C"), |
| }; |
| |
| // Build list of supported OpenCL C features |
| m_opencl_c_features = { |
| MAKE_NAME_VERSION(3, 0, 0, "__opencl_c_images"), |
| MAKE_NAME_VERSION(3, 0, 0, "__opencl_c_3d_image_writes"), |
| // TODO(#216) re-enable when clspv ready |
| // MAKE_NAME_VERSION(3, 0, 0, "__opencl_c_generic_address_space"), |
| }; |
| if (supports_read_write_images()) { |
| m_opencl_c_features.push_back( |
| MAKE_NAME_VERSION(3, 0, 0, "__opencl_c_read_write_images")); |
| } |
| if (supports_atomic_order_acq_rel()) { |
| m_opencl_c_features.push_back( |
| MAKE_NAME_VERSION(3, 0, 0, "__opencl_c_atomic_order_acq_rel")); |
| } |
| if (supports_atomic_scope_device()) { |
| m_opencl_c_features.push_back( |
| MAKE_NAME_VERSION(3, 0, 0, "__opencl_c_atomic_scope_device")); |
| } |
| if (supports_subgroups()) { |
| m_opencl_c_features.push_back( |
| MAKE_NAME_VERSION(3, 0, 0, "__opencl_c_subgroups")); |
| } |
| if (m_features.features.shaderInt64) { |
| m_opencl_c_features.push_back( |
| MAKE_NAME_VERSION(3, 0, 0, "__opencl_c_int64")); |
| } |
| if (supports_fp64()) { |
| m_opencl_c_features.push_back( |
| MAKE_NAME_VERSION(3, 0, 0, "__opencl_c_fp64")); |
| } |
| if (supports_dot_product()) { |
| if (supports_int8()) { |
| m_opencl_c_features.push_back(MAKE_NAME_VERSION( |
| 3, 0, 0, "__opencl_c_integer_dot_product_input_4x8bit")); |
| } |
| m_opencl_c_features.push_back(MAKE_NAME_VERSION( |
| 3, 0, 0, "__opencl_c_integer_dot_product_input_4x8bit_packed")); |
| } |
| } |
| |
| bool cvk_device::create_vulkan_queues_and_device(uint32_t num_queues, |
| uint32_t queue_family) { |
| cvk_info("Creating Vulkan device and queues"); |
| // Give all queues the same priority |
| std::vector<float> queuePriorities(num_queues, 1.0f); |
| void* pNext = nullptr; |
| VkDeviceQueueGlobalPriorityCreateInfoKHR globalPriorityCreateInfo; |
| cvk_info_fn("queue global priority: %u", |
| m_features_queue_global_priority.globalPriorityQuery); |
| if (m_features_queue_global_priority.globalPriorityQuery) { |
| const uint32_t num_priorities = 4; |
| static const VkQueueGlobalPriorityKHR priorities[num_priorities] = { |
| VK_QUEUE_GLOBAL_PRIORITY_LOW_KHR, |
| VK_QUEUE_GLOBAL_PRIORITY_MEDIUM_KHR, |
| VK_QUEUE_GLOBAL_PRIORITY_HIGH_KHR, |
| VK_QUEUE_GLOBAL_PRIORITY_REALTIME_KHR}; |
| uint32_t queue_priority = |
| std::min(config.queue_global_priority(), num_priorities - 1); |
| |
| globalPriorityCreateInfo.sType = |
| VK_STRUCTURE_TYPE_DEVICE_QUEUE_GLOBAL_PRIORITY_CREATE_INFO_KHR; |
| globalPriorityCreateInfo.pNext = nullptr; |
| globalPriorityCreateInfo.globalPriority = priorities[queue_priority]; |
| pNext = &globalPriorityCreateInfo; |
| |
| cvk_info_fn("setting queue global priority to '%u': '%s' (%u)", |
| queue_priority, |
| queue_global_priority_to_string( |
| globalPriorityCreateInfo.globalPriority), |
| globalPriorityCreateInfo.globalPriority); |
| } |
| |
| VkDeviceQueueCreateInfo queueCreateInfo = { |
| VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, |
| pNext, |
| 0, // flags |
| queue_family, |
| num_queues, // queueCount |
| queuePriorities.data()}; |
| |
| // Create logical device |
| const VkDeviceCreateInfo createInfo = { |
| VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, // sType |
| &m_features, // pNext |
| 0, // flags |
| 1, // queueCreateInfoCount |
| &queueCreateInfo, // pQueueCreateInfos, |
| 0, // enabledLayerCount |
| nullptr, // ppEnabledLayerNames |
| static_cast<uint32_t>( |
| m_vulkan_device_extensions.size()), // enabledExtensionCount |
| m_vulkan_device_extensions.data(), // ppEnabledExtensionNames |
| nullptr, // pEnabledFeatures |
| }; |
| |
| VkResult res = vkCreateDevice(m_pdev, &createInfo, nullptr, &m_dev); |
| CVK_VK_CHECK_ERROR_RET(res, false, "Failed to create a device"); |
| |
| // Construct the queue wrappers now that our queues exist |
| m_vulkan_queues.reserve(num_queues); |
| for (auto i = 0U; i < num_queues; i++) { |
| VkQueue queue; |
| |
| vkGetDeviceQueue(m_dev, queue_family, i, &queue); |
| m_vulkan_queues.emplace_back(queue, queue_family); |
| } |
| |
| return true; |
| } |
| |
| bool cvk_device::init_time_management(VkInstance instance) { |
| |
| if (is_vulkan_extension_enabled( |
| VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME)) { |
| auto func = GET_INSTANCE_PROC( |
| instance, vkGetPhysicalDeviceCalibrateableTimeDomainsEXT); |
| uint32_t num_time_domains; |
| VkResult res; |
| res = func(m_pdev, &num_time_domains, nullptr); |
| if (res != VK_SUCCESS) { |
| cvk_error( |
| "Can't get number of available calibrateable time domains"); |
| return false; |
| } |
| |
| cvk_info("Device supports %u calibrateable time domains", |
| num_time_domains); |
| |
| std::vector<VkTimeDomainEXT> supported_time_domains(num_time_domains); |
| res = func(m_pdev, &num_time_domains, supported_time_domains.data()); |
| if (res != VK_SUCCESS) { |
| cvk_error("Can't get list of available calibrateable time domains"); |
| return false; |
| } |
| |
| bool has_device = false; |
| bool has_monotonic = false; |
| for (auto td : supported_time_domains) { |
| cvk_info(" %s", |
| vulkan_calibrateable_time_domain_string(td).c_str()); |
| if (td == VK_TIME_DOMAIN_CLOCK_MONOTONIC_EXT) { |
| has_monotonic = true; |
| } |
| |
| if (td == VK_TIME_DOMAIN_DEVICE_EXT) { |
| has_device = true; |
| } |
| } |
| |
| if (has_device && has_monotonic) { |
| m_has_timer_support = true; |
| m_vkfns.vkGetCalibratedTimestampsEXT = |
| GET_INSTANCE_PROC(instance, vkGetCalibratedTimestampsEXT); |
| } |
| } |
| |
| if (!m_has_timer_support) { |
| cvk_warn("This device does not support VK_EXT_calibrated_timestamps or " |
| "it does not support the required " |
| "VK_TIME_DOMAIN_CLOCK_MONOTONIC_EXT and " |
| "VK_TIME_DOMAIN_DEVICE_EXT time domains"); |
| cvk_warn("clGetHostTimer and clGetDeviceAndHostTimer will not work"); |
| cvk_warn("Command queue profiling will suffer from limitations"); |
| } |
| |
| return true; |
| } |
| |
| // Returns the pipeline cache file path for a given SPIR-V SHA-1 hash. |
| // If pipeline cache serialization is not enabled, an empty string is returned. |
| std::string |
| cvk_device::get_pipeline_cache_filename(const cvk_sha1_hash& sha1) const { |
| if (config.cache_dir().empty()) { |
| return ""; |
| } |
| |
| // The pipeline cache file path is: |
| // ${CLVK_CACHE_DIR}/clvk-pipeline-cache.<UUID>.<SHA1>.bin |
| std::string cache_path = config.cache_dir; |
| cache_path += "/"; |
| cache_path += "clvk-pipeline-cache."; |
| cache_path += to_hex_string(m_properties.pipelineCacheUUID, VK_UUID_SIZE); |
| cache_path += "."; |
| cache_path += to_hex_string(reinterpret_cast<const uint8_t*>(sha1.data()), |
| SHA1_DIGEST_NUM_BYTES); |
| cache_path += ".bin"; |
| return cache_path; |
| } |
| |
| bool cvk_device::get_pipeline_cache(const std::vector<uint32_t>& spirv, |
| VkPipelineCache& pipeline_cache) { |
| |
| std::lock_guard<std::mutex> lock(m_pipeline_cache_mutex); |
| |
| pipeline_cache = VK_NULL_HANDLE; |
| |
| // Compute SHA-1 hash of the SPIR-V binary |
| cvk_sha1_hash sha1 = |
| cvk_sha1(spirv.data(), spirv.size() * sizeof(uint32_t)); |
| |
| // Check the in-memory cache of pipeline caches |
| if (m_pipeline_caches.count(sha1)) { |
| pipeline_cache = m_pipeline_caches.at(sha1); |
| return true; |
| } |
| |
| std::vector<char> cache_data; |
| |
| // Load pipeline cache data from file if this is enabled |
| std::string cache_path = get_pipeline_cache_filename(sha1); |
| if (!cache_path.empty()) { |
| cvk_info("Looking for pipeline cache at %s", cache_path.c_str()); |
| std::ifstream cache_file(cache_path, std::ios::in | std::ios::binary); |
| if (cache_file.is_open()) { |
| // Get the size of the pipeline cache file |
| cache_file.seekg(0, std::ios::end); |
| uint32_t size = cache_file.tellg(); |
| cache_file.seekg(0, std::ios::beg); |
| |
| // Load the pipeline cache data into memory |
| cache_data.resize(size); |
| cache_file.read(cache_data.data(), size); |
| if (!cache_file.good()) { |
| cvk_warn("Failed to read pipeline cache data"); |
| cache_data.clear(); |
| } |
| } else { |
| cvk_warn("Failed to open pipeline cache file"); |
| } |
| } |
| |
| // Create pipeline cache |
| VkPipelineCacheCreateInfo pipelineCacheCreateInfo = { |
| VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, |
| nullptr, // pNext |
| 0, // flags |
| cache_data.size(), // initialDataSize |
| cache_data.data(), // pInitialData |
| }; |
| |
| VkResult res = vkCreatePipelineCache(m_dev, &pipelineCacheCreateInfo, |
| nullptr, &pipeline_cache); |
| if (res != VK_SUCCESS) { |
| cvk_error("Could not create pipeline cache."); |
| return false; |
| } |
| |
| // Add pipeline cache to the in-memory cache |
| m_pipeline_caches[sha1] = pipeline_cache; |
| |
| return cache_data.size() != 0; |
| } |
| |
| void cvk_device::save_pipeline_cache( |
| const cvk_sha1_hash& sha1, const VkPipelineCache& pipeline_cache) const { |
| VkResult res; |
| |
| std::string cache_path = get_pipeline_cache_filename(sha1); |
| if (cache_path.empty()) { |
| return; |
| } |
| |
| // Retrieve the pipeline cache data from the Vulkan implementation |
| size_t size; |
| res = vkGetPipelineCacheData(m_dev, pipeline_cache, &size, nullptr); |
| if (res != VK_SUCCESS) { |
| cvk_error("Failed to retrieve pipeline cache size"); |
| return; |
| } |
| std::vector<char> cache_data(size); |
| res = |
| vkGetPipelineCacheData(m_dev, pipeline_cache, &size, cache_data.data()); |
| if (res != VK_SUCCESS) { |
| cvk_error("Failed to retrieve pipeline cache data"); |
| return; |
| } |
| |
| cvk_info("Writing %lu bytes of pipeline cache data to file", size); |
| |
| // Write the pipeline cache data to file |
| std::ofstream cache_file(cache_path, std::ios::out | std::ios::binary); |
| if (!cache_file.is_open()) { |
| cvk_error("Failed to open pipeline cache file for writing: %s", |
| cache_path.c_str()); |
| return; |
| } |
| cache_file.write(cache_data.data(), size); |
| if (!cache_file.good()) { |
| cvk_error("Failed to write pipeline cache data"); |
| } |
| } |
| |
| void cvk_device::init_spirv_environment() { |
| if (m_properties.apiVersion < VK_MAKE_VERSION(1, 1, 0)) { |
| m_vulkan_spirv_env = SPV_ENV_VULKAN_1_0; |
| } else if (m_properties.apiVersion < VK_MAKE_VERSION(1, 2, 0)) { |
| if (is_vulkan_extension_enabled(VK_KHR_SPIRV_1_4_EXTENSION_NAME)) { |
| m_vulkan_spirv_env = SPV_ENV_VULKAN_1_1_SPIRV_1_4; |
| } else { |
| m_vulkan_spirv_env = SPV_ENV_VULKAN_1_1; |
| } |
| } else if (m_properties.apiVersion < VK_MAKE_VERSION(1, 3, 0)) { |
| m_vulkan_spirv_env = SPV_ENV_VULKAN_1_2; |
| } else { |
| // Assume 1.3 |
| m_vulkan_spirv_env = SPV_ENV_VULKAN_1_3; |
| } |
| cvk_info("Vulkan SPIR-V environment: %s", |
| spvTargetEnvDescription(m_vulkan_spirv_env)); |
| } |
| |
| void cvk_device::log_limits_and_memory_information() { |
| // Print relevant device limits |
| const VkPhysicalDeviceLimits& limits = vulkan_limits(); |
| cvk_info("Device's resources per stage limits:"); |
| cvk_info(" total = %u", limits.maxPerStageResources); |
| cvk_info(" uniform buffers = %u", |
| limits.maxPerStageDescriptorUniformBuffers); |
| cvk_info(" storage buffers = %u", |
| limits.maxPerStageDescriptorStorageBuffers); |
| cvk_info("Device's max buffer size = %s", |
| pretty_size(limits.maxStorageBufferRange).c_str()); |
| cvk_info("Device's max uniform buffer size = %s", |
| pretty_size(limits.maxUniformBufferRange).c_str()); |
| cvk_info("Device's max push constant size = %s", |
| pretty_size(limits.maxPushConstantsSize).c_str()); |
| cvk_info("Device's execution capabilities:"); |
| cvk_info(" Max work-group count: {%u,%u,%u}", |
| limits.maxComputeWorkGroupCount[0], |
| limits.maxComputeWorkGroupCount[1], |
| limits.maxComputeWorkGroupCount[2]); |
| cvk_info(" Max invocations per work-group: %u", |
| limits.maxComputeWorkGroupInvocations); |
| cvk_info(" Max work-group size: {%u,%u,%u}", |
| limits.maxComputeWorkGroupSize[0], |
| limits.maxComputeWorkGroupSize[1], |
| limits.maxComputeWorkGroupSize[2]); |
| cvk_info(" Min subgroup size: %u", |
| m_subgroup_size_control_properties.minSubgroupSize); |
| cvk_info(" Max subgroup size: %u", |
| m_subgroup_size_control_properties.maxSubgroupSize); |
| cvk_info(" Max subgroups per work-group: %u", |
| m_subgroup_size_control_properties.maxComputeWorkgroupSubgroups); |
| |
| // Print memoy information |
| cvk_info("Device has %u memory types:", m_mem_properties.memoryTypeCount); |
| for (uint32_t i = 0; i < m_mem_properties.memoryTypeCount; i++) { |
| VkMemoryType memtype = m_mem_properties.memoryTypes[i]; |
| auto heapsize = m_mem_properties.memoryHeaps[memtype.heapIndex].size; |
| cvk_info( |
| " %u: heap = %u, %s | %s", i, memtype.heapIndex, |
| pretty_size(heapsize).c_str(), |
| vulkan_memory_property_flags_string(memtype.propertyFlags).c_str()); |
| } |
| |
| cvk_info("Device has %u memory heaps:", m_mem_properties.memoryHeapCount); |
| for (uint32_t i = 0; i < m_mem_properties.memoryHeapCount; i++) { |
| VkMemoryHeap memheap = m_mem_properties.memoryHeaps[i]; |
| cvk_info(" %u: %s | %s", i, pretty_size(memheap.size).c_str(), |
| vulkan_memory_property_flags_string(memheap.flags).c_str()); |
| } |
| } |
| |
| bool cvk_device::init(VkInstance instance) { |
| cvk_info("Initialising device %s", m_properties.deviceName); |
| cvk_info(" API Version: %s", |
| vulkan_version_string(m_properties.apiVersion).c_str()); |
| |
| uint32_t num_queues, queue_family; |
| if (!init_queues(&num_queues, &queue_family)) { |
| return false; |
| } |
| |
| if (!init_extensions()) { |
| return false; |
| } |
| |
| init_vulkan_properties(instance); |
| |
| init_clvk_runtime_behaviors(); |
| |
| init_driver_behaviors(); |
| |
| init_features(instance); |
| |
| init_command_pointers(instance); |
| |
| build_extension_ils_list(); |
| |
| if (!init_time_management(instance)) { |
| return false; |
| } |
| |
| if (!create_vulkan_queues_and_device(num_queues, queue_family)) { |
| return false; |
| } |
| |
| init_spirv_environment(); |
| |
| log_limits_and_memory_information(); |
| |
| // Must be done last as it relies on info set up in several of the above. |
| init_compiler_options(); |
| |
| return true; |
| } |
| |
| std::string cvk_device::vendor() const { |
| // Is this a Khronos vendor ID? |
| if (m_properties.vendorID > 0xFFFF) { |
| return vulkan_vendor_id_string( |
| static_cast<VkVendorId>(m_properties.vendorID)); |
| } |
| |
| // If not, we are looking at a PCI Vendor ID. |
| // TODO support looking it up in the PCI DB. |
| |
| return m_clvk_properties->vendor(); |
| } |
| |
| bool cvk_device::supports_capability(spv::Capability capability) const { |
| switch (capability) { |
| // Capabilities required by all Vulkan implementations: |
| case spv::CapabilityShader: |
| case spv::CapabilitySampled1D: |
| case spv::CapabilityImage1D: |
| case spv::CapabilityImageQuery: |
| case spv::CapabilityImageBuffer: |
| case spv::CapabilitySampledBuffer: |
| return true; |
| // Optional capabilities: |
| case spv::CapabilityFloat16: |
| return m_features_float16_int8.shaderFloat16; |
| case spv::CapabilityFloat64: |
| return m_features.features.shaderFloat64; |
| case spv::CapabilityInt8: |
| return m_features_float16_int8.shaderInt8; |
| case spv::CapabilityInt16: |
| return m_features.features.shaderInt16; |
| case spv::CapabilityInt64: |
| return m_features.features.shaderInt64; |
| case spv::CapabilityStorageImageReadWithoutFormat: |
| return m_features.features.shaderStorageImageReadWithoutFormat; |
| case spv::CapabilityStorageImageWriteWithoutFormat: |
| return m_features.features.shaderStorageImageWriteWithoutFormat; |
| case spv::CapabilityVariablePointers: |
| return m_features_variable_pointer.variablePointers; |
| case spv::CapabilityVariablePointersStorageBuffer: |
| return m_features_variable_pointer.variablePointersStorageBuffer; |
| case spv::CapabilityGroupNonUniform: |
| return m_subgroup_properties.supportedOperations & |
| VK_SUBGROUP_FEATURE_BASIC_BIT; |
| case spv::CapabilityGroupNonUniformVote: |
| return m_subgroup_properties.supportedOperations & |
| VK_SUBGROUP_FEATURE_VOTE_BIT; |
| case spv::CapabilityGroupNonUniformArithmetic: |
| return m_subgroup_properties.supportedOperations & |
| VK_SUBGROUP_FEATURE_ARITHMETIC_BIT; |
| case spv::CapabilityGroupNonUniformBallot: |
| return m_subgroup_properties.supportedOperations & |
| VK_SUBGROUP_FEATURE_BALLOT_BIT; |
| case spv::CapabilityGroupNonUniformShuffle: |
| return m_subgroup_properties.supportedOperations & |
| VK_SUBGROUP_FEATURE_SHUFFLE_BIT; |
| case spv::CapabilityGroupNonUniformRotateKHR: |
| return supports_subgroup_rotate(); |
| case spv::CapabilityVulkanMemoryModel: |
| return m_features_vulkan_memory_model.vulkanMemoryModel; |
| case spv::CapabilityShaderNonUniform: |
| return supports_non_uniform_decoration(); |
| case spv::CapabilityPhysicalStorageBufferAddresses: |
| return m_features_buffer_device_address.bufferDeviceAddress; |
| case spv::CapabilityRoundingModeRTE: |
| return m_float_controls_properties.shaderRoundingModeRTEFloat32 || |
| m_float_controls_properties.shaderRoundingModeRTEFloat16 || |
| m_float_controls_properties.shaderRoundingModeRTEFloat64; |
| case spv::CapabilityDotProduct: |
| case spv::CapabilityDotProductInput4x8BitPacked: |
| return supports_dot_product(); |
| case spv::CapabilityDotProductInput4x8Bit: |
| case spv::CapabilityDotProductInputAll: |
| return supports_dot_product() && supports_int8(); |
| // Capabilities that have not yet been mapped to Vulkan features: |
| default: |
| cvk_warn_fn("Capability %d not yet mapped to a feature.", capability); |
| return false; |
| } |
| } |
| |
| void cvk_device::select_work_group_size( |
| cvk_kernel* kernel, const std::array<uint32_t, 3>& global_size, |
| std::array<uint32_t, 3>& local_size) const { |
| |
| auto required_work_group_size = kernel->required_work_group_size(); |
| if (required_work_group_size[0] != 0) { |
| local_size = required_work_group_size; |
| return; |
| } |
| // Start at (1,1,1), which is always valid. |
| local_size = {1, 1, 1}; |
| |
| // Cap the total work-group size to the Vulkan device's limit. |
| uint32_t max_size = m_properties.limits.maxComputeWorkGroupInvocations; |
| |
| // Further cap the total size to 64, as this is expected to be a |
| // reasonable size on many devices. |
| max_size = std::min(max_size, UINT32_C(64)); |
| |
| // TODO: We should also take into account the total number of |
| // work-groups that would be launched, to ensure the device is fully |
| // utilized for smaller global work sizes. |
| |
| // Increase the work-group size until we hit device limits. |
| bool changed; |
| do { |
| changed = false; |
| |
| // Alternate between increasing the X and Y dimensions. |
| // TODO: Consider increasing the Z dimension as well? |
| for (int i = 0; i < 2; i++) { |
| // Double the dimension if we can. |
| // TODO: Allow non power-of-two sizes? |
| // TODO: Allow non-uniform sizes if supported? |
| std::array<uint32_t, 3> new_local_size = local_size; |
| new_local_size[i] *= 2; |
| if (global_size[i] % new_local_size[i] == 0 && |
| new_local_size[i] <= |
| m_properties.limits.maxComputeWorkGroupCount[i] && |
| new_local_size[0] * new_local_size[1] * new_local_size[2] <= |
| max_size) { |
| local_size = new_local_size; |
| changed = true; |
| } |
| } |
| } while (changed); |
| } |
| |
| cl_int cvk_device::get_device_host_timer(cl_ulong* device_timestamp, |
| cl_ulong* host_timestamp) const { |
| auto vkdev = vulkan_device(); |
| |
| uint64_t timestamps[2]; |
| uint64_t max_deviation; |
| VkCalibratedTimestampInfoEXT timestamp_infos[2] = { |
| {VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_EXT, nullptr, |
| VK_TIME_DOMAIN_CLOCK_MONOTONIC_EXT}, |
| {VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_EXT, nullptr, |
| VK_TIME_DOMAIN_DEVICE_EXT}}; |
| |
| uint32_t num_requested_timestamps; |
| if (device_timestamp == nullptr) { |
| num_requested_timestamps = 1; |
| } else { |
| num_requested_timestamps = 2; |
| } |
| |
| auto res = m_vkfns.vkGetCalibratedTimestampsEXT( |
| vkdev, num_requested_timestamps, timestamp_infos, timestamps, |
| &max_deviation); |
| if (res != VK_SUCCESS) { |
| cvk_error_fn("vkGetCalibratedTimestampsEXT failed %d %s", res, |
| vulkan_error_string(res)); |
| return CL_OUT_OF_RESOURCES; |
| } |
| |
| *host_timestamp = timestamps[0]; |
| if (device_timestamp != nullptr) { |
| *device_timestamp = timestamp_to_ns(timestamps[1]); |
| } |
| |
| return CL_SUCCESS; |
| } |
| |
| cl_int cvk_device::update_device_host_timer_no_lock() { |
| return get_device_host_timer(&m_sync_dev, &m_sync_host); |
| } |
| |
| cl_int cvk_device::update_device_host_timer() { |
| std::lock_guard<std::mutex> lock(m_sync_mutex); |
| return update_device_host_timer_no_lock(); |
| } |
| |
| cl_int cvk_device::device_timer_to_host(cl_ulong dev, cl_ulong& host) { |
| std::lock_guard<std::mutex> lock(m_sync_mutex); |
| if (dev > m_sync_dev) { |
| cl_int err = update_device_host_timer_no_lock(); |
| if (err != CL_SUCCESS) { |
| return err; |
| } |
| } |
| if (m_sync_host > m_sync_dev) { |
| host = (m_sync_host - m_sync_dev) + dev; |
| } else { |
| host = dev - (m_sync_dev - m_sync_host); |
| } |
| return CL_SUCCESS; |
| } |
