blob: 9353b56d326d2fcf5a34a76fddbd00c836455e9d [file] [log] [blame]
//
// Copyright 2018 Pierre Moreau
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
//
#include "invocation.hpp"
#include <tuple>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#ifdef HAVE_CLOVER_SPIRV
#include <spirv-tools/libspirv.hpp>
#include <spirv-tools/linker.hpp>
#endif
#include "core/error.hpp"
#include "core/platform.hpp"
#include "invocation.hpp"
#include "llvm/util.hpp"
#include "pipe/p_state.h"
#include "util/algorithm.hpp"
#include "util/functional.hpp"
#include "util/u_math.h"
#include "compiler/spirv/spirv.h"
#define SPIRV_HEADER_WORD_SIZE 5
using namespace clover;
#ifdef HAVE_CLOVER_SPIRV
namespace {
template<typename T>
T get(const char *source, size_t index) {
const uint32_t *word_ptr = reinterpret_cast<const uint32_t *>(source);
return static_cast<T>(word_ptr[index]);
}
enum module::argument::type
convert_storage_class(SpvStorageClass storage_class, std::string &err) {
switch (storage_class) {
case SpvStorageClassFunction:
return module::argument::scalar;
case SpvStorageClassUniformConstant:
return module::argument::global;
case SpvStorageClassWorkgroup:
return module::argument::local;
case SpvStorageClassCrossWorkgroup:
return module::argument::global;
default:
err += "Invalid storage type " + std::to_string(storage_class) + "\n";
throw build_error();
}
}
enum module::argument::type
convert_image_type(SpvId id, SpvDim dim, SpvAccessQualifier access,
std::string &err) {
if (dim == SpvDim2D && access == SpvAccessQualifierReadOnly)
return module::argument::image2d_rd;
else if (dim == SpvDim2D && access == SpvAccessQualifierWriteOnly)
return module::argument::image2d_wr;
else if (dim == SpvDim3D && access == SpvAccessQualifierReadOnly)
return module::argument::image3d_rd;
else if (dim == SpvDim3D && access == SpvAccessQualifierWriteOnly)
return module::argument::image3d_wr;
else {
err += "Unknown access qualifier " + std::to_string(access)
+ " or dimension " + std::to_string(dim) + " for image "
+ std::to_string(id) + ".\n";
throw build_error();
}
}
module::section
make_text_section(const std::vector<char> &code,
enum module::section::type section_type) {
const pipe_binary_program_header header { uint32_t(code.size()) };
module::section text { 0, section_type, header.num_bytes, {} };
text.data.insert(text.data.end(), reinterpret_cast<const char *>(&header),
reinterpret_cast<const char *>(&header) + sizeof(header));
text.data.insert(text.data.end(), code.begin(), code.end());
return text;
}
module
create_module_from_spirv(const std::vector<char> &source,
size_t pointer_byte_size,
std::string &err) {
const size_t length = source.size() / sizeof(uint32_t);
size_t i = SPIRV_HEADER_WORD_SIZE; // Skip header
std::string kernel_name;
size_t kernel_nb = 0u;
std::vector<module::argument> args;
module m;
std::unordered_map<SpvId, std::string> kernels;
std::unordered_map<SpvId, module::argument> types;
std::unordered_map<SpvId, SpvId> pointer_types;
std::unordered_map<SpvId, unsigned int> constants;
std::unordered_set<SpvId> packed_structures;
std::unordered_map<SpvId, std::vector<SpvFunctionParameterAttribute>>
func_param_attr_map;
while (i < length) {
const auto inst = &source[i * sizeof(uint32_t)];
const auto desc_word = get<uint32_t>(inst, 0);
const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask);
const unsigned int num_operands = desc_word >> SpvWordCountShift;
switch (opcode) {
case SpvOpEntryPoint:
if (get<SpvExecutionModel>(inst, 1) == SpvExecutionModelKernel)
kernels.emplace(get<SpvId>(inst, 2),
source.data() + (i + 3u) * sizeof(uint32_t));
break;
case SpvOpDecorate: {
const auto id = get<SpvId>(inst, 1);
const auto decoration = get<SpvDecoration>(inst, 2);
if (decoration == SpvDecorationCPacked)
packed_structures.emplace(id);
else if (decoration == SpvDecorationFuncParamAttr) {
const auto attribute =
get<SpvFunctionParameterAttribute>(inst, 3u);
func_param_attr_map[id].push_back(attribute);
}
break;
}
case SpvOpGroupDecorate: {
const auto group_id = get<SpvId>(inst, 1);
if (packed_structures.count(group_id)) {
for (unsigned int i = 2u; i < num_operands; ++i)
packed_structures.emplace(get<SpvId>(inst, i));
}
const auto func_param_attr_iter =
func_param_attr_map.find(group_id);
if (func_param_attr_iter != func_param_attr_map.end()) {
for (unsigned int i = 2u; i < num_operands; ++i)
func_param_attr_map.emplace(get<SpvId>(inst, i),
func_param_attr_iter->second);
}
break;
}
case SpvOpConstant:
// We only care about constants that represent the size of arrays.
// If they are passed as argument, they will never be more than
// 4GB-wide, and even if they did, a clover::module::argument size
// is represented by an int.
constants[get<SpvId>(inst, 2)] = get<unsigned int>(inst, 3u);
break;
case SpvOpTypeInt: // FALLTHROUGH
case SpvOpTypeFloat: {
const auto size = get<uint32_t>(inst, 2) / 8u;
types[get<SpvId>(inst, 1)] = { module::argument::scalar, size,
size, size,
module::argument::zero_ext };
break;
}
case SpvOpTypeArray: {
const auto id = get<SpvId>(inst, 1);
const auto type_id = get<SpvId>(inst, 2);
const auto types_iter = types.find(type_id);
if (types_iter == types.end())
break;
const auto constant_id = get<SpvId>(inst, 3);
const auto constants_iter = constants.find(constant_id);
if (constants_iter == constants.end()) {
err += "Constant " + std::to_string(constant_id) +
" is missing\n";
throw build_error();
}
const auto elem_size = types_iter->second.size;
const auto elem_nbs = constants_iter->second;
const auto size = elem_size * elem_nbs;
types[id] = { module::argument::scalar, size, size,
types_iter->second.target_align,
module::argument::zero_ext };
break;
}
case SpvOpTypeStruct: {
const auto id = get<SpvId>(inst, 1);
const bool is_packed = packed_structures.count(id);
unsigned struct_size = 0u;
unsigned struct_align = 1u;
for (unsigned j = 2u; j < num_operands; ++j) {
const auto type_id = get<SpvId>(inst, j);
const auto types_iter = types.find(type_id);
// If a type was not found, that means it is not one of the
// types allowed as kernel arguments. And since the module has
// been validated, this means this type is not used for kernel
// arguments, and therefore can be ignored.
if (types_iter == types.end())
break;
const auto alignment = is_packed ? 1u
: types_iter->second.target_align;
const auto padding = (-struct_size) & (alignment - 1u);
struct_size += padding + types_iter->second.target_size;
struct_align = std::max(struct_align, alignment);
}
struct_size += (-struct_size) & (struct_align - 1u);
types[id] = { module::argument::scalar, struct_size, struct_size,
struct_align, module::argument::zero_ext };
break;
}
case SpvOpTypeVector: {
const auto id = get<SpvId>(inst, 1);
const auto type_id = get<SpvId>(inst, 2);
const auto types_iter = types.find(type_id);
// If a type was not found, that means it is not one of the
// types allowed as kernel arguments. And since the module has
// been validated, this means this type is not used for kernel
// arguments, and therefore can be ignored.
if (types_iter == types.end())
break;
const auto elem_size = types_iter->second.size;
const auto elem_nbs = get<uint32_t>(inst, 3);
const auto size = elem_size * elem_nbs;
types[id] = { module::argument::scalar, size, size, size,
module::argument::zero_ext };
break;
}
case SpvOpTypeForwardPointer: // FALLTHROUGH
case SpvOpTypePointer: {
const auto id = get<SpvId>(inst, 1);
const auto storage_class = get<SpvStorageClass>(inst, 2);
// Input means this is for a builtin variable, which can not be
// passed as an argument to a kernel.
if (storage_class == SpvStorageClassInput)
break;
types[id] = { convert_storage_class(storage_class, err),
sizeof(cl_mem),
static_cast<module::size_t>(pointer_byte_size),
static_cast<module::size_t>(pointer_byte_size),
module::argument::zero_ext };
if (opcode == SpvOpTypePointer)
pointer_types[id] = get<SpvId>(inst, 3);
break;
}
case SpvOpTypeSampler:
types[get<SpvId>(inst, 1)] = { module::argument::sampler,
sizeof(cl_sampler) };
break;
case SpvOpTypeImage: {
const auto id = get<SpvId>(inst, 1);
const auto dim = get<SpvDim>(inst, 3);
const auto access = get<SpvAccessQualifier>(inst, 9);
types[id] = { convert_image_type(id, dim, access, err),
sizeof(cl_mem), sizeof(cl_mem), sizeof(cl_mem),
module::argument::zero_ext };
break;
}
case SpvOpTypePipe: // FALLTHROUGH
case SpvOpTypeQueue: {
err += "TypePipe and TypeQueue are valid SPIR-V 1.0 types, but are "
"not available in the currently supported OpenCL C version."
"\n";
throw build_error();
}
case SpvOpFunction: {
const auto kernels_iter = kernels.find(get<SpvId>(inst, 2));
if (kernels_iter != kernels.end())
kernel_name = kernels_iter->second;
break;
}
case SpvOpFunctionParameter: {
if (kernel_name.empty())
break;
const auto type_id = get<SpvId>(inst, 1);
auto arg = types.find(type_id)->second;
const auto &func_param_attr_iter =
func_param_attr_map.find(get<SpvId>(inst, 2));
if (func_param_attr_iter != func_param_attr_map.end()) {
for (auto &i : func_param_attr_iter->second) {
switch (i) {
case SpvFunctionParameterAttributeSext:
arg.ext_type = module::argument::sign_ext;
break;
case SpvFunctionParameterAttributeZext:
arg.ext_type = module::argument::zero_ext;
break;
case SpvFunctionParameterAttributeByVal: {
const SpvId ptr_type_id =
pointer_types.find(type_id)->second;
arg = types.find(ptr_type_id)->second;
break;
}
default:
break;
}
}
}
args.emplace_back(arg);
break;
}
case SpvOpFunctionEnd:
if (kernel_name.empty())
break;
m.syms.emplace_back(kernel_name, 0, kernel_nb, args);
++kernel_nb;
kernel_name.clear();
args.clear();
break;
default:
break;
}
i += num_operands;
}
m.secs.push_back(make_text_section(source,
module::section::text_intermediate));
return m;
}
bool
check_capabilities(const device &dev, const std::vector<char> &source,
std::string &r_log) {
const size_t length = source.size() / sizeof(uint32_t);
size_t i = SPIRV_HEADER_WORD_SIZE; // Skip header
while (i < length) {
const auto desc_word = get<uint32_t>(source.data(), i);
const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask);
const unsigned int num_operands = desc_word >> SpvWordCountShift;
if (opcode != SpvOpCapability)
break;
const auto capability = get<SpvCapability>(source.data(), i + 1u);
switch (capability) {
// Mandatory capabilities
case SpvCapabilityAddresses:
case SpvCapabilityFloat16Buffer:
case SpvCapabilityGroups:
case SpvCapabilityInt64:
case SpvCapabilityInt16:
case SpvCapabilityInt8:
case SpvCapabilityKernel:
case SpvCapabilityLinkage:
case SpvCapabilityVector16:
break;
// Optional capabilities
case SpvCapabilityImageBasic:
case SpvCapabilityLiteralSampler:
case SpvCapabilitySampled1D:
case SpvCapabilityImage1D:
case SpvCapabilitySampledBuffer:
case SpvCapabilityImageBuffer:
if (!dev.image_support()) {
r_log += "Capability 'ImageBasic' is not supported.\n";
return false;
}
break;
case SpvCapabilityFloat64:
if (!dev.has_doubles()) {
r_log += "Capability 'Float64' is not supported.\n";
return false;
}
break;
// Enabled through extensions
case SpvCapabilityFloat16:
if (!dev.has_halves()) {
r_log += "Capability 'Float16' is not supported.\n";
return false;
}
break;
case SpvCapabilityInt64Atomics:
if (!dev.has_int64_atomics()) {
r_log += "Capability 'Int64Atomics' is not supported.\n";
return false;
}
break;
default:
r_log += "Capability '" + std::to_string(capability) +
"' is not supported.\n";
return false;
}
i += num_operands;
}
return true;
}
bool
check_extensions(const device &dev, const std::vector<char> &source,
std::string &r_log) {
const size_t length = source.size() / sizeof(uint32_t);
size_t i = SPIRV_HEADER_WORD_SIZE; // Skip header
const auto spirv_extensions = spirv::supported_extensions();
while (i < length) {
const auto desc_word = get<uint32_t>(source.data(), i);
const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask);
const unsigned int num_operands = desc_word >> SpvWordCountShift;
if (opcode == SpvOpCapability) {
i += num_operands;
continue;
}
if (opcode != SpvOpExtension)
break;
const std::string extension = source.data() + (i + 1u) * sizeof(uint32_t);
if (spirv_extensions.count(extension) == 0) {
r_log += "Extension '" + extension + "' is not supported.\n";
return false;
}
i += num_operands;
}
return true;
}
bool
check_memory_model(const device &dev, const std::vector<char> &source,
std::string &r_log) {
const size_t length = source.size() / sizeof(uint32_t);
size_t i = SPIRV_HEADER_WORD_SIZE; // Skip header
while (i < length) {
const auto desc_word = get<uint32_t>(source.data(), i);
const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask);
const unsigned int num_operands = desc_word >> SpvWordCountShift;
switch (opcode) {
case SpvOpMemoryModel:
switch (get<SpvAddressingModel>(source.data(), i + 1u)) {
case SpvAddressingModelPhysical32:
return dev.address_bits() == 32;
case SpvAddressingModelPhysical64:
return dev.address_bits() == 64;
default:
unreachable("Only Physical32 and Physical64 are valid for OpenCL, and the binary was already validated");
return false;
}
break;
default:
break;
}
i += num_operands;
}
return false;
}
// Copies the input binary and convert it to the endianness of the host CPU.
std::vector<char>
spirv_to_cpu(const std::vector<char> &binary)
{
const uint32_t first_word = get<uint32_t>(binary.data(), 0u);
if (first_word == SpvMagicNumber)
return binary;
std::vector<char> cpu_endianness_binary(binary.size());
for (size_t i = 0; i < (binary.size() / 4u); ++i) {
const uint32_t word = get<uint32_t>(binary.data(), i);
reinterpret_cast<uint32_t *>(cpu_endianness_binary.data())[i] =
util_bswap32(word);
}
return cpu_endianness_binary;
}
#ifdef HAVE_CLOVER_SPIRV
std::string
format_validator_msg(spv_message_level_t level, const char * /* source */,
const spv_position_t &position, const char *message) {
std::string level_str;
switch (level) {
case SPV_MSG_FATAL:
level_str = "Fatal";
break;
case SPV_MSG_INTERNAL_ERROR:
level_str = "Internal error";
break;
case SPV_MSG_ERROR:
level_str = "Error";
break;
case SPV_MSG_WARNING:
level_str = "Warning";
break;
case SPV_MSG_INFO:
level_str = "Info";
break;
case SPV_MSG_DEBUG:
level_str = "Debug";
break;
}
return "[" + level_str + "] At word No." +
std::to_string(position.index) + ": \"" + message + "\"\n";
}
spv_target_env
convert_opencl_str_to_target_env(const std::string &opencl_version) {
if (opencl_version == "2.2") {
return SPV_ENV_OPENCL_2_2;
} else if (opencl_version == "2.1") {
return SPV_ENV_OPENCL_2_1;
} else if (opencl_version == "2.0") {
return SPV_ENV_OPENCL_2_0;
} else if (opencl_version == "1.2" ||
opencl_version == "1.1" ||
opencl_version == "1.0") {
// SPIR-V is only defined for OpenCL >= 1.2, however some drivers
// might use it with OpenCL 1.0 and 1.1.
return SPV_ENV_OPENCL_1_2;
} else {
throw build_error("Invalid OpenCL version");
}
}
#endif
}
module
clover::spirv::compile_program(const std::vector<char> &binary,
const device &dev, std::string &r_log) {
std::vector<char> source = spirv_to_cpu(binary);
if (!is_valid_spirv(source, dev.device_version(), r_log))
throw build_error();
if (!check_capabilities(dev, source, r_log))
throw build_error();
if (!check_extensions(dev, source, r_log))
throw build_error();
if (!check_memory_model(dev, source, r_log))
throw build_error();
return create_module_from_spirv(source,
dev.address_bits() == 32 ? 4u : 8u, r_log);
}
module
clover::spirv::link_program(const std::vector<module> &modules,
const device &dev, const std::string &opts,
std::string &r_log) {
std::vector<std::string> options = clover::llvm::tokenize(opts);
bool create_library = false;
std::string ignored_options;
for (const std::string &option : options) {
if (option == "-create-library") {
create_library = true;
} else {
ignored_options += "'" + option + "' ";
}
}
if (!ignored_options.empty()) {
r_log += "Ignoring the following link options: " + ignored_options
+ "\n";
}
spvtools::LinkerOptions linker_options;
linker_options.SetCreateLibrary(create_library);
module m;
const auto section_type = create_library ? module::section::text_library :
module::section::text_executable;
std::vector<const uint32_t *> sections;
sections.reserve(modules.size());
std::vector<size_t> lengths;
lengths.reserve(modules.size());
auto const validator_consumer = [&r_log](spv_message_level_t level,
const char *source,
const spv_position_t &position,
const char *message) {
r_log += format_validator_msg(level, source, position, message);
};
for (const auto &mod : modules) {
const auto &msec = find([](const module::section &sec) {
return sec.type == module::section::text_intermediate ||
sec.type == module::section::text_library;
}, mod.secs);
const auto c_il = ((struct pipe_binary_program_header*)msec.data.data())->blob;
const auto length = msec.size;
sections.push_back(reinterpret_cast<const uint32_t *>(c_il));
lengths.push_back(length / sizeof(uint32_t));
}
std::vector<uint32_t> linked_binary;
const std::string opencl_version = dev.device_version();
const spv_target_env target_env =
convert_opencl_str_to_target_env(opencl_version);
const spvtools::MessageConsumer consumer = validator_consumer;
spvtools::Context context(target_env);
context.SetMessageConsumer(std::move(consumer));
if (Link(context, sections.data(), lengths.data(), sections.size(),
&linked_binary, linker_options) != SPV_SUCCESS)
throw error(CL_LINK_PROGRAM_FAILURE);
std::vector<char> final_binary{
reinterpret_cast<char *>(linked_binary.data()),
reinterpret_cast<char *>(linked_binary.data() +
linked_binary.size()) };
if (!is_valid_spirv(final_binary, opencl_version, r_log))
throw error(CL_LINK_PROGRAM_FAILURE);
for (const auto &mod : modules)
m.syms.insert(m.syms.end(), mod.syms.begin(), mod.syms.end());
m.secs.emplace_back(make_text_section(final_binary, section_type));
return m;
}
bool
clover::spirv::is_valid_spirv(const std::vector<char> &binary,
const std::string &opencl_version,
std::string &r_log) {
auto const validator_consumer =
[&r_log](spv_message_level_t level, const char *source,
const spv_position_t &position, const char *message) {
r_log += format_validator_msg(level, source, position, message);
};
const spv_target_env target_env =
convert_opencl_str_to_target_env(opencl_version);
spvtools::SpirvTools spvTool(target_env);
spvTool.SetMessageConsumer(validator_consumer);
return spvTool.Validate(reinterpret_cast<const uint32_t *>(binary.data()),
binary.size() / 4u);
}
std::string
clover::spirv::print_module(const std::vector<char> &binary,
const std::string &opencl_version) {
const spv_target_env target_env =
convert_opencl_str_to_target_env(opencl_version);
spvtools::SpirvTools spvTool(target_env);
spv_context spvContext = spvContextCreate(target_env);
if (!spvContext)
return "Failed to create an spv_context for disassembling the module.";
spv_text disassembly;
spvBinaryToText(spvContext,
reinterpret_cast<const uint32_t *>(binary.data()),
binary.size() / 4u, SPV_BINARY_TO_TEXT_OPTION_NONE,
&disassembly, nullptr);
spvContextDestroy(spvContext);
const std::string disassemblyStr = disassembly->str;
spvTextDestroy(disassembly);
return disassemblyStr;
}
std::unordered_set<std::string>
clover::spirv::supported_extensions() {
return {
/* this is only a hint so all devices support that */
"SPV_KHR_no_integer_wrap_decoration"
};
}
#else
bool
clover::spirv::is_valid_spirv(const std::vector<char> &/*binary*/,
const std::string &/*opencl_version*/,
std::string &/*r_log*/) {
return false;
}
module
clover::spirv::compile_program(const std::vector<char> &binary,
const device &dev, std::string &r_log) {
r_log += "SPIR-V support in clover is not enabled.\n";
throw build_error();
}
module
clover::spirv::link_program(const std::vector<module> &/*modules*/,
const device &/*dev*/, const std::string &/*opts*/,
std::string &r_log) {
r_log += "SPIR-V support in clover is not enabled.\n";
throw error(CL_LINKER_NOT_AVAILABLE);
}
std::string
clover::spirv::print_module(const std::vector<char> &binary,
const std::string &opencl_version) {
return std::string();
}
std::unordered_set<std::string>
clover::spirv::supported_extensions() {
return {};
}
#endif