blob: 5fac2a24151d9bf8cb40c52ec7d1a0c141a2688f [file] [log] [blame]
// SPDX-License-Identifier: GPL-2.0
/*
* Utility functions of mailbox protocol for Edge TPU ML accelerator.
*
* Copyright (C) 2019 Google, Inc.
*/
#include <asm/page.h>
#include <linux/bitops.h>
#include <linux/bits.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/mmzone.h> /* MAX_ORDER_NR_PAGES */
#include <linux/slab.h>
#include "edgetpu-device-group.h"
#include "edgetpu-iremap-pool.h"
#include "edgetpu-kci.h"
#include "edgetpu-mailbox.h"
#include "edgetpu-mmu.h"
#include "edgetpu-sw-watchdog.h"
#include "edgetpu-wakelock.h"
#include "edgetpu.h"
/*
* Checks if @size is a valid circular queue size, which should be a positive
* number and less than or equal to MAX_QUEUE_SIZE.
*/
static inline bool valid_circular_queue_size(u32 size)
{
if (!size || size > MAX_QUEUE_SIZE)
return false;
return true;
}
/* Return context ID for mailbox. */
static inline enum edgetpu_context_id
edgetpu_mailbox_context_id(struct edgetpu_mailbox *mailbox)
{
if (!mailbox)
return EDGETPU_CONTEXT_INVALID;
return EDGETPU_CONTEXT_VII_BASE + mailbox->mailbox_id - 1;
}
/* Sets mailbox->cmd_queue_tail and corresponding CSR on device. */
static void edgetpu_mailbox_set_cmd_queue_tail(struct edgetpu_mailbox *mailbox,
u32 value)
{
mailbox->cmd_queue_tail = value;
EDGETPU_MAILBOX_CMD_QUEUE_WRITE_SYNC(mailbox, tail, value);
}
/* Sets mailbox->resp_queue_head and corresponding CSR on device. */
static void edgetpu_mailbox_set_resp_queue_head(struct edgetpu_mailbox *mailbox,
u32 value)
{
mailbox->resp_queue_head = value;
EDGETPU_MAILBOX_RESP_QUEUE_WRITE(mailbox, head, value);
}
/*
* Allocates and returns a mailbox given the index of this mailbox,
* also enables the mailbox.
*
* Caller holds mgr->mailboxes_lock.
*/
static struct edgetpu_mailbox *
edgetpu_mailbox_create_locked(struct edgetpu_mailbox_manager *mgr, uint index)
{
struct edgetpu_mailbox *mailbox = kzalloc(sizeof(*mailbox), GFP_ATOMIC);
if (!mailbox)
return ERR_PTR(-ENOMEM);
mailbox->mailbox_id = index;
mailbox->etdev = mgr->etdev;
mailbox->context_csr_base = mgr->get_context_csr_base(index);
mailbox->cmd_queue_csr_base = mgr->get_cmd_queue_csr_base(index);
mailbox->resp_queue_csr_base = mgr->get_resp_queue_csr_base(index);
edgetpu_mailbox_init_doorbells(mailbox);
return mailbox;
}
/* Caller must hold @mgr->mailboxes_lock. */
static int edgetpu_mailbox_remove_locked(struct edgetpu_mailbox_manager *mgr,
struct edgetpu_mailbox *mailbox)
{
/* simple security checks */
if (mailbox->mailbox_id >= mgr->num_mailbox ||
mgr->mailboxes[mailbox->mailbox_id] != mailbox) {
return -EINVAL;
}
mgr->mailboxes[mailbox->mailbox_id] = NULL;
/* KCI mailbox is a special case */
if (mailbox->mailbox_id == KERNEL_MAILBOX_INDEX)
edgetpu_kci_release(mgr->etdev, mailbox->internal.kci);
kfree(mailbox);
return 0;
}
/*
* Disables the @index-th mailbox via setting CSR. Doesn't need
* @mgr->mailboxes[index] be allocated.
*/
static void edgetpu_mailbox_disable_ith(struct edgetpu_mailbox_manager *mgr,
uint index)
{
struct edgetpu_mailbox mbox = {
.etdev = mgr->etdev,
.context_csr_base = mgr->get_context_csr_base(index),
};
edgetpu_mailbox_disable(&mbox);
}
static void edgetpu_vii_irq_handler(struct edgetpu_mailbox *mailbox)
{
if (mailbox->internal.group)
edgetpu_group_notify(mailbox->internal.group,
EDGETPU_EVENT_RESPDATA);
}
/*
* Increases the command queue tail by @inc.
*
* The queue uses the mirrored circular buffer arrangement. Each index (head and
* tail) has a wrap bit, represented by the constant CIRCULAR_QUEUE_WRAP_BIT.
* Whenever an index is increased and will exceed the end of the queue, the wrap
* bit is xor-ed.
*
* This method will update both mailbox->cmd_queue_tail and CSR on device.
*
* Caller ensures @inc is less than the space remain in the command queue.
*/
void edgetpu_mailbox_inc_cmd_queue_tail(struct edgetpu_mailbox *mailbox,
u32 inc)
{
u32 new_tail;
new_tail = circular_queue_inc(mailbox->cmd_queue_tail, inc,
mailbox->cmd_queue_size);
edgetpu_mailbox_set_cmd_queue_tail(mailbox, new_tail);
}
/*
* Increases the response queue head by @inc.
*
* The queue uses the mirrored circular buffer arrangement. Each index (head and
* tail) has a wrap bit, represented by the constant CIRCULAR_QUEUE_WRAP_BIT.
* Whenever an index is increased and will exceed the end of the queue, the wrap
* bit is xor-ed.
*
* This method will update both mailbox->resp_queue_head and CSR on device.
*
* Caller ensures @inc is less than the distance between resp_head and
* resp_tail.
*/
void edgetpu_mailbox_inc_resp_queue_head(struct edgetpu_mailbox *mailbox,
u32 inc)
{
u32 new_head;
new_head = circular_queue_inc(mailbox->resp_queue_head, inc,
mailbox->resp_queue_size);
edgetpu_mailbox_set_resp_queue_head(mailbox, new_head);
}
/*
* Sets address and size of queue.
*
* Sets the queue address with @addr, a 36-bit address, and with size @size in
* units of number of elements.
*
* Returns 0 on success.
* -EINVAL is returned if @addr or @size is invalid.
*/
int edgetpu_mailbox_set_queue(struct edgetpu_mailbox *mailbox,
enum mailbox_queue_type type, u64 addr, u32 size)
{
u32 low = addr & 0xffffffff;
u32 high = addr >> 32;
if (!valid_circular_queue_size(size))
return -EINVAL;
/* addr is a 36-bit address, checks if the higher bits are clear */
if (high & 0xfffffff0)
return -EINVAL;
switch (type) {
case MAILBOX_CMD_QUEUE:
EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, cmd_queue_address_low,
low);
EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, cmd_queue_address_high,
high);
EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, cmd_queue_size, size);
mailbox->cmd_queue_size = size;
edgetpu_mailbox_set_cmd_queue_tail(mailbox, 0);
EDGETPU_MAILBOX_CMD_QUEUE_WRITE(mailbox, head, 0);
break;
case MAILBOX_RESP_QUEUE:
EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, resp_queue_address_low,
low);
EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, resp_queue_address_high,
high);
EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, resp_queue_size, size);
mailbox->resp_queue_size = size;
edgetpu_mailbox_set_resp_queue_head(mailbox, 0);
EDGETPU_MAILBOX_RESP_QUEUE_WRITE(mailbox, tail, 0);
break;
}
return 0;
}
/* Reset mailbox queues, clear out any commands/responses left from before. */
void edgetpu_mailbox_reset(struct edgetpu_mailbox *mailbox)
{
edgetpu_mailbox_disable(mailbox);
EDGETPU_MAILBOX_CMD_QUEUE_WRITE(mailbox, head, 0);
edgetpu_mailbox_set_cmd_queue_tail(mailbox, 0);
edgetpu_mailbox_set_resp_queue_head(mailbox, 0);
EDGETPU_MAILBOX_RESP_QUEUE_WRITE(mailbox, tail, 0);
edgetpu_mailbox_enable(mailbox);
}
/* Sets the priority of @mailbox. */
void edgetpu_mailbox_set_priority(struct edgetpu_mailbox *mailbox, u32 priority)
{
EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, priority, priority);
}
struct edgetpu_mailbox *
edgetpu_mailbox_vii_add(struct edgetpu_mailbox_manager *mgr, uint id)
{
struct edgetpu_mailbox *mailbox = NULL;
unsigned long flags;
write_lock_irqsave(&mgr->mailboxes_lock, flags);
if (id == 0) {
uint i;
for (i = mgr->vii_index_from; i < mgr->vii_index_to; i++) {
if (!mgr->mailboxes[i]) {
id = i;
break;
}
}
} else {
/* no mailbox available - returns busy */
if (id < mgr->vii_index_from || id >= mgr->vii_index_to ||
mgr->mailboxes[id])
id = 0;
}
/* no empty slot found */
if (id == 0) {
mailbox = ERR_PTR(-EBUSY);
} else {
mailbox = edgetpu_mailbox_create_locked(mgr, id);
if (!IS_ERR(mailbox)) {
mgr->mailboxes[id] = mailbox;
mailbox->handle_irq = edgetpu_vii_irq_handler;
}
}
write_unlock_irqrestore(&mgr->mailboxes_lock, flags);
return mailbox;
}
/*
* Requests the first @n P2P mailboxes, where @n should be the number of devices
* in a virtual device group.
*
* The array of requested mailboxes will be assigned to @mailboxes on success.
* @mailboxes must have dimension @n, @mailboxes[@skip_i] will be set to NULL.
*
* Returns 0 on success, or a negative errno on error.
* Returns -EBUSY if any mailbox is using.
*
* Caller calls edgetpu_mailbox_enable() to enable the returned mailboxes.
*/
int edgetpu_mailbox_p2p_batch(struct edgetpu_mailbox_manager *mgr, uint n,
uint skip_i, struct edgetpu_mailbox **mailboxes)
{
uint i, j;
int ret;
struct edgetpu_mailbox *mailbox;
unsigned long flags;
if (mgr->p2p_index_to - mgr->p2p_index_from < n)
return -EINVAL;
write_lock_irqsave(&mgr->mailboxes_lock, flags);
memset(mailboxes, 0, sizeof(*mailboxes) * n);
for (i = mgr->p2p_index_from, j = 0; j < n; i++, j++) {
if (mgr->mailboxes[i]) {
ret = -EBUSY;
goto release;
}
}
for (i = mgr->p2p_index_from, j = 0; j < n; i++, j++) {
if (j == skip_i) {
edgetpu_mailbox_disable_ith(mgr, i);
continue;
}
mailbox = edgetpu_mailbox_create_locked(mgr, i);
if (IS_ERR(mailbox)) {
ret = PTR_ERR(mailbox);
goto release;
}
mailboxes[j] = mgr->mailboxes[i] = mailbox;
}
write_unlock_irqrestore(&mgr->mailboxes_lock, flags);
return 0;
release:
write_unlock_irqrestore(&mgr->mailboxes_lock, flags);
for (i = 0; i < n; i++) {
if (mailboxes[i])
edgetpu_mailbox_remove(mgr, mailboxes[i]);
mailboxes[i] = NULL;
}
return ret;
}
/*
* Every mailbox manager can allocate one mailbox for KCI to use.
* -EBUSY is returned if the KCI mailbox is allocated and hasn't been removed
* via edgetpu_mailbox_remove().
*/
struct edgetpu_mailbox *edgetpu_mailbox_kci(struct edgetpu_mailbox_manager *mgr)
{
struct edgetpu_mailbox *mailbox;
unsigned long flags;
write_lock_irqsave(&mgr->mailboxes_lock, flags);
if (mgr->mailboxes[KERNEL_MAILBOX_INDEX]) {
mailbox = ERR_PTR(-EBUSY);
goto out;
}
mailbox = edgetpu_mailbox_create_locked(mgr, KERNEL_MAILBOX_INDEX);
if (!IS_ERR(mailbox))
mgr->mailboxes[KERNEL_MAILBOX_INDEX] = mailbox;
out:
write_unlock_irqrestore(&mgr->mailboxes_lock, flags);
return mailbox;
}
/*
* Removes a mailbox from the manager.
* Returns 0 on success.
*/
int edgetpu_mailbox_remove(struct edgetpu_mailbox_manager *mgr, struct edgetpu_mailbox *mailbox)
{
unsigned long flags;
int ret;
write_lock_irqsave(&mgr->mailboxes_lock, flags);
ret = edgetpu_mailbox_remove_locked(mgr, mailbox);
write_unlock_irqrestore(&mgr->mailboxes_lock, flags);
return ret;
}
/*
* The queue size of edgetpu_mailbox_attr has units in KB, convert it to use the
* element size here.
*
* Returns a negative errno on error, or the converted size.
*/
static int convert_runtime_queue_size_to_fw(u32 queue_size, u32 element_size)
{
const u32 runtime_unit = 1024;
u32 ret;
/* zero size is not allowed */
if (queue_size == 0 || element_size == 0)
return -EINVAL;
/* A quick check to prevent the queue allocation failure. */
if (queue_size > (MAX_ORDER_NR_PAGES << PAGE_SHIFT) / runtime_unit)
return -ENOMEM;
/*
* Kernel doesn't care whether queue_size * runtime_unit is a multiple
* of element_size.
*/
ret = queue_size * runtime_unit / element_size;
/* hardware limitation */
if (ret == 0 || ret > MAX_QUEUE_SIZE)
return -EINVAL;
return ret;
}
int edgetpu_mailbox_validate_attr(const struct edgetpu_mailbox_attr *attr)
{
int size;
size = convert_runtime_queue_size_to_fw(attr->cmd_queue_size,
attr->sizeof_cmd);
if (size < 0)
return size;
size = convert_runtime_queue_size_to_fw(attr->resp_queue_size,
attr->sizeof_resp);
if (size < 0)
return size;
return 0;
}
int edgetpu_mailbox_init_vii(struct edgetpu_vii *vii,
struct edgetpu_device_group *group)
{
int cmd_queue_size, resp_queue_size;
struct edgetpu_mailbox_manager *mgr = group->etdev->mailbox_manager;
struct edgetpu_mailbox *mailbox;
const struct edgetpu_mailbox_attr *attr = &group->mbox_attr;
int ret;
if (!group->etdomain || group->etdomain->pasid == IOMMU_PASID_INVALID)
mailbox = edgetpu_mailbox_vii_add(mgr, 0);
else
mailbox = edgetpu_mailbox_vii_add(mgr, group->etdomain->pasid);
if (IS_ERR(mailbox))
return PTR_ERR(mailbox);
cmd_queue_size = convert_runtime_queue_size_to_fw(attr->cmd_queue_size,
attr->sizeof_cmd);
resp_queue_size = convert_runtime_queue_size_to_fw(
attr->resp_queue_size, attr->sizeof_resp);
edgetpu_mailbox_set_priority(mailbox, attr->priority);
EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox,
cmd_queue_tail_doorbell_enable,
attr->cmdq_tail_doorbell);
ret = edgetpu_mailbox_alloc_queue(group->etdev, mailbox, cmd_queue_size,
attr->sizeof_cmd, MAILBOX_CMD_QUEUE,
&vii->cmd_queue_mem);
if (ret) {
edgetpu_mailbox_remove(mgr, mailbox);
return ret;
}
etdev_dbg(group->etdev,
"%s: mbox %u cmdq iova=%#llx dma=%pad\n",
__func__, mailbox->mailbox_id, vii->cmd_queue_mem.tpu_addr,
&vii->cmd_queue_mem.dma_addr);
ret = edgetpu_mailbox_alloc_queue(group->etdev, mailbox,
resp_queue_size, attr->sizeof_resp,
MAILBOX_RESP_QUEUE,
&vii->resp_queue_mem);
if (ret) {
edgetpu_mailbox_free_queue(group->etdev, mailbox,
&vii->cmd_queue_mem);
edgetpu_mailbox_remove(mgr, mailbox);
return ret;
}
etdev_dbg(group->etdev,
"%s: mbox %u rspq iova=%#llx dma=%pad\n",
__func__, mailbox->mailbox_id, vii->resp_queue_mem.tpu_addr,
&vii->resp_queue_mem.dma_addr);
mailbox->internal.group = edgetpu_device_group_get(group);
vii->etdev = group->etdev;
vii->mailbox = mailbox;
edgetpu_mailbox_enable(mailbox);
return 0;
}
void edgetpu_mailbox_remove_vii(struct edgetpu_vii *vii)
{
struct edgetpu_dev *etdev;
etdev = vii->etdev;
edgetpu_mailbox_free_queue(etdev, vii->mailbox, &vii->cmd_queue_mem);
edgetpu_mailbox_free_queue(etdev, vii->mailbox, &vii->resp_queue_mem);
if (vii->mailbox) {
if (!vii->mailbox->internal.group->dev_inaccessible)
edgetpu_mailbox_disable(vii->mailbox);
edgetpu_device_group_put(vii->mailbox->internal.group);
edgetpu_mailbox_remove(etdev->mailbox_manager, vii->mailbox);
vii->mailbox = NULL;
}
}
static int edgetpu_mailbox_do_alloc_queue(struct edgetpu_dev *etdev,
struct edgetpu_mailbox *mailbox, u32 queue_size,
u32 unit, edgetpu_queue_mem *mem)
{
u32 size = unit * queue_size;
/* Align queue size to page size for TPU MMU map. */
size = __ALIGN_KERNEL(size, PAGE_SIZE);
return edgetpu_iremap_alloc(etdev, size, mem,
edgetpu_mailbox_context_id(mailbox));
}
/*
* Allocates memory for a queue.
*
* The total size (in bytes) of queue is @queue_size * @unit.
* CSRs of @mailbox include queue_size and queue_address will be set on success.
* @mem->dma_addr, @mem->vaddr, and @mem->size will be set.
*
* Returns 0 on success, or a negative errno on error.
*/
int edgetpu_mailbox_alloc_queue(struct edgetpu_dev *etdev,
struct edgetpu_mailbox *mailbox, u32 queue_size,
u32 unit, enum mailbox_queue_type type,
edgetpu_queue_mem *mem)
{
int ret = edgetpu_mailbox_do_alloc_queue(etdev, mailbox, queue_size, unit, mem);
if (ret)
return ret;
ret = edgetpu_mailbox_set_queue(mailbox, type, mem->tpu_addr,
queue_size);
if (ret) {
edgetpu_mailbox_free_queue(etdev, mailbox, mem);
return ret;
}
return 0;
}
/*
* Releases the queue memory previously allocated with
* edgetpu_mailbox_alloc_queue().
*
* Does nothing if @mem->vaddr is NULL.
*/
void edgetpu_mailbox_free_queue(struct edgetpu_dev *etdev,
struct edgetpu_mailbox *mailbox,
edgetpu_queue_mem *mem)
{
if (!mem->vaddr)
return;
edgetpu_iremap_free(etdev, mem, edgetpu_mailbox_context_id(mailbox));
}
/*
* Creates a mailbox manager, one edgetpu device has one manager.
*/
struct edgetpu_mailbox_manager *
edgetpu_mailbox_create_mgr(struct edgetpu_dev *etdev,
const struct edgetpu_mailbox_manager_desc *desc)
{
struct edgetpu_mailbox_manager *mgr;
uint total = 0;
total += 1; /* KCI mailbox */
total += desc->num_vii_mailbox;
total += desc->num_p2p_mailbox;
total += desc->num_ext_mailbox;
if (total > desc->num_mailbox)
return ERR_PTR(-EINVAL);
mgr = devm_kzalloc(etdev->dev, sizeof(*mgr), GFP_KERNEL);
if (!mgr)
return ERR_PTR(-ENOMEM);
mgr->etdev = etdev;
mgr->num_mailbox = desc->num_mailbox;
/* index 0 is reserved for KCI */
mgr->vii_index_from = 1;
mgr->vii_index_to = mgr->vii_index_from + desc->num_vii_mailbox;
mgr->p2p_index_from = mgr->vii_index_to;
mgr->p2p_index_to = mgr->p2p_index_from + desc->num_p2p_mailbox;
mgr->ext_index_from = mgr->p2p_index_to;
mgr->ext_index_to = mgr->ext_index_from + desc->num_ext_mailbox;
mgr->get_context_csr_base = desc->get_context_csr_base;
mgr->get_cmd_queue_csr_base = desc->get_cmd_queue_csr_base;
mgr->get_resp_queue_csr_base = desc->get_resp_queue_csr_base;
mgr->mailboxes = devm_kcalloc(etdev->dev, mgr->num_mailbox,
sizeof(*mgr->mailboxes), GFP_KERNEL);
if (!mgr->mailboxes)
return ERR_PTR(-ENOMEM);
rwlock_init(&mgr->mailboxes_lock);
mutex_init(&mgr->open_devices.lock);
return mgr;
}
/* All requested mailboxes will be disabled and freed. */
void edgetpu_mailbox_remove_all(struct edgetpu_mailbox_manager *mgr)
{
uint i;
struct edgetpu_mailbox *kci_mailbox = NULL;
unsigned long flags;
if (IS_ERR_OR_NULL(mgr))
return;
write_lock_irqsave(&mgr->mailboxes_lock, flags);
for (i = 0; i < mgr->num_mailbox; i++) {
struct edgetpu_mailbox *mailbox = mgr->mailboxes[i];
if (mailbox) {
edgetpu_mailbox_disable(mailbox);
/* KCI needs special handling */
if (i == KERNEL_MAILBOX_INDEX)
kci_mailbox = mailbox;
else
kfree(mailbox);
mgr->mailboxes[i] = NULL;
}
}
write_unlock_irqrestore(&mgr->mailboxes_lock, flags);
/* Cancel KCI worker outside the lock, then free KCI mailbox. */
if (kci_mailbox) {
edgetpu_kci_release(mgr->etdev,
kci_mailbox->internal.kci);
kfree(kci_mailbox);
}
}
/*
* The interrupt handler for KCI and VII mailboxes.
*
* This handler loops through such mailboxes with an interrupt pending and
* invokes their IRQ handlers.
*/
irqreturn_t edgetpu_mailbox_handle_irq(struct edgetpu_mailbox_manager *mgr)
{
struct edgetpu_mailbox *mailbox;
uint i;
if (!mgr)
return IRQ_NONE;
read_lock(&mgr->mailboxes_lock);
for (i = 0; i < mgr->vii_index_to; i++) {
mailbox = mgr->mailboxes[i];
if (!mailbox)
continue;
if (!EDGETPU_MAILBOX_RESP_QUEUE_READ(mailbox, doorbell_status))
continue;
EDGETPU_MAILBOX_RESP_QUEUE_WRITE(mailbox, doorbell_clear, 1);
etdev_dbg(mgr->etdev, "mbox %u resp doorbell irq tail=%u\n",
i, EDGETPU_MAILBOX_RESP_QUEUE_READ(mailbox, tail));
if (mailbox->handle_irq)
mailbox->handle_irq(mailbox);
}
read_unlock(&mgr->mailboxes_lock);
return IRQ_HANDLED;
}
void edgetpu_mailbox_init_doorbells(struct edgetpu_mailbox *mailbox)
{
/* Clear any stale doorbells requested */
EDGETPU_MAILBOX_RESP_QUEUE_WRITE(mailbox, doorbell_clear, 1);
EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, cmd_queue_doorbell_clear, 1);
/* Enable the command and response doorbell interrupts */
EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, cmd_queue_doorbell_enable, 1);
EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, resp_queue_doorbell_enable, 1);
}
void edgetpu_mailbox_reset_mailboxes(struct edgetpu_mailbox_manager *mgr)
{
uint i;
unsigned long flags;
write_lock_irqsave(&mgr->mailboxes_lock, flags);
/*
* Reset all the allocated mailboxes, starting from VII till
* external mailboxes.
*/
for (i = mgr->vii_index_from; i < mgr->ext_index_to; i++) {
struct edgetpu_mailbox *mbox = mgr->mailboxes[i];
if (!mbox)
continue;
edgetpu_mailbox_reset(mbox);
edgetpu_mailbox_disable(mbox);
edgetpu_mailbox_init_doorbells(mbox);
}
write_unlock_irqrestore(&mgr->mailboxes_lock, flags);
}
void edgetpu_mailbox_reinit_vii(struct edgetpu_device_group *group)
{
int cmd_queue_size, resp_queue_size;
struct edgetpu_mailbox *mailbox = group->vii.mailbox;
const struct edgetpu_mailbox_attr *attr = &group->mbox_attr;
cmd_queue_size = convert_runtime_queue_size_to_fw(attr->cmd_queue_size,
attr->sizeof_cmd);
resp_queue_size = convert_runtime_queue_size_to_fw(
attr->resp_queue_size, attr->sizeof_resp);
etdev_dbg(group->etdev, "Restoring vii. workload_id=%u mbox_id=%u\n",
group->workload_id, mailbox->mailbox_id);
etdev_dbg(group->etdev, "Priority: %d\n", attr->priority);
etdev_dbg(group->etdev, "Tail doorbell %s",
attr->cmdq_tail_doorbell ? "enabled" : "disabled");
etdev_dbg(group->etdev, "cmd queue: addr=%llX size=%u\n",
group->vii.cmd_queue_mem.tpu_addr, cmd_queue_size);
etdev_dbg(group->etdev, "resp queue: addr=%llX size=%u\n",
group->vii.resp_queue_mem.tpu_addr, resp_queue_size);
edgetpu_mailbox_set_priority(mailbox, attr->priority);
EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, cmd_queue_tail_doorbell_enable,
attr->cmdq_tail_doorbell);
edgetpu_mailbox_set_queue(mailbox, MAILBOX_CMD_QUEUE,
group->vii.cmd_queue_mem.tpu_addr,
cmd_queue_size);
edgetpu_mailbox_set_queue(mailbox, MAILBOX_RESP_QUEUE,
group->vii.resp_queue_mem.tpu_addr,
resp_queue_size);
edgetpu_mailbox_enable(mailbox);
}
static void edgetpu_mailbox_init_external_mailbox(struct edgetpu_external_mailbox *ext_mailbox)
{
struct edgetpu_mailbox_attr attr;
struct edgetpu_mailbox *mailbox;
struct edgetpu_mailbox_descriptor *desc;
uint i;
attr = ext_mailbox->attr;
for (i = 0; i < ext_mailbox->count; i++) {
desc = &ext_mailbox->descriptors[i];
mailbox = desc->mailbox;
edgetpu_mailbox_set_priority(mailbox, attr.priority);
EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, cmd_queue_tail_doorbell_enable,
attr.cmdq_tail_doorbell);
edgetpu_mailbox_set_queue(mailbox, MAILBOX_CMD_QUEUE,
desc->cmd_queue_mem.tpu_addr,
attr.cmd_queue_size);
edgetpu_mailbox_set_queue(mailbox, MAILBOX_RESP_QUEUE,
desc->resp_queue_mem.tpu_addr,
attr.resp_queue_size);
edgetpu_mailbox_enable(mailbox);
}
}
void edgetpu_mailbox_reinit_external_mailbox(struct edgetpu_device_group *group)
{
struct edgetpu_external_mailbox *ext_mailbox = group->ext_mailbox;
if (!ext_mailbox)
return;
etdev_dbg(group->etdev, "Restoring external attached %d mailboxes\n", ext_mailbox->count);
edgetpu_mailbox_init_external_mailbox(ext_mailbox);
}
void edgetpu_mailbox_restore_active_mailbox_queues(struct edgetpu_dev *etdev)
{
struct edgetpu_list_group *l;
struct edgetpu_device_group *group;
struct edgetpu_device_group **groups;
size_t i, n = 0;
mutex_lock(&etdev->groups_lock);
groups = kmalloc_array(etdev->n_groups, sizeof(*groups), GFP_KERNEL);
if (unlikely(!groups)) {
/*
* Either the runtime is misbehaving (creates tons of groups),
* or the system is indeed OOM - we give up this restore
* process, which makes the runtime unable to communicate with
* the device through VII.
*/
mutex_unlock(&etdev->groups_lock);
return;
}
/*
* Fetch the groups into an array to restore the VII without holding
* etdev->groups_lock. To prevent the potential deadlock that
* edgetpu_device_group_add() holds group->lock then etdev->groups_lock.
*/
etdev_for_each_group(etdev, l, group) {
/*
* Quick skip without holding group->lock.
* Disbanded groups can never go back to the normal state.
*/
if (edgetpu_device_group_is_disbanded(group))
continue;
/*
* Increase the group reference to prevent the group being
* released after we release groups_lock.
*/
groups[n++] = edgetpu_device_group_get(group);
}
mutex_unlock(&etdev->groups_lock);
/*
* We are not holding @etdev->groups_lock, what may race is:
* 1. The group is disbanding and being removed from @etdev.
* 2. A new group is adding to @etdev
*
* For (1.) the group will be marked as DISBANDED, so we check whether
* the group is finalized before performing VII re-init.
*
* For (2.), adding group to @etdev (edgetpu_device_group_add()) has
* nothing to do with VII, its VII will be set when the group is
* finalized.
*/
for (i = 0; i < n; i++) {
group = groups[i];
mutex_lock(&group->lock);
/*
* If the group is just finalized or has mailbox attached in
* another process, this re-init is redundant but isn't harmful.
*/
if (edgetpu_group_finalized_and_attached(group)) {
edgetpu_mailbox_reinit_vii(group);
edgetpu_mailbox_reinit_external_mailbox(group);
}
mutex_unlock(&group->lock);
edgetpu_device_group_put(group);
}
kfree(groups);
}
static int edgetpu_mailbox_external_alloc_queue_batch(struct edgetpu_external_mailbox *ext_mailbox)
{
int ret, i;
struct edgetpu_mailbox *mailbox;
struct edgetpu_mailbox_attr attr;
struct edgetpu_mailbox_descriptor *desc;
struct edgetpu_dev *etdev = ext_mailbox->etdev;
attr = ext_mailbox->attr;
for (i = 0; i < ext_mailbox->count; i++) {
desc = &ext_mailbox->descriptors[i];
mailbox = desc->mailbox;
ret = edgetpu_mailbox_do_alloc_queue(etdev, mailbox, attr.cmd_queue_size,
attr.sizeof_cmd, &desc->cmd_queue_mem);
if (ret)
goto undo;
ret = edgetpu_mailbox_do_alloc_queue(etdev, mailbox, attr.resp_queue_size,
attr.sizeof_resp, &desc->resp_queue_mem);
if (ret) {
edgetpu_mailbox_free_queue(etdev, mailbox,
&desc->cmd_queue_mem);
goto undo;
}
}
return 0;
undo:
while (i--) {
desc = &ext_mailbox->descriptors[i];
mailbox = desc->mailbox;
edgetpu_mailbox_free_queue(etdev, mailbox, &desc->cmd_queue_mem);
edgetpu_mailbox_free_queue(etdev, mailbox, &desc->resp_queue_mem);
}
return ret;
}
static void edgetpu_mailbox_external_free_queue_batch(struct edgetpu_external_mailbox *ext_mailbox)
{
u32 i;
struct edgetpu_mailbox *mailbox;
struct edgetpu_mailbox_descriptor *desc;
struct edgetpu_dev *etdev = ext_mailbox->etdev;
for (i = 0; i < ext_mailbox->count; i++) {
desc = &ext_mailbox->descriptors[i];
mailbox = desc->mailbox;
edgetpu_mailbox_free_queue(etdev, mailbox, &desc->cmd_queue_mem);
edgetpu_mailbox_free_queue(etdev, mailbox, &desc->resp_queue_mem);
}
}
/*
* Checks if the indexes given for external mailboxes are in range of mailbox
* manager(@mgr) managing the external mailboxes.
*/
static bool edgetpu_mailbox_external_check_range(struct edgetpu_mailbox_manager *mgr,
const int start, const int end)
{
return (start <= end) && (mgr->ext_index_from <= start && mgr->ext_index_to > end);
}
/*
* Allocates external mailboxes according to @ext_mailbox_req object and
* associate it with @group.
*
* Caller should hold @group->lock
*/
static int edgetpu_mailbox_external_alloc(struct edgetpu_device_group *group,
struct edgetpu_external_mailbox_req *ext_mailbox_req)
{
u32 i, j = 0, bmap, start, end;
struct edgetpu_mailbox_manager *mgr = group->etdev->mailbox_manager;
struct edgetpu_mailbox *mailbox;
int ret = 0, count;
struct edgetpu_external_mailbox *ext_mailbox;
struct edgetpu_mailbox_attr attr;
unsigned long flags;
if (!edgetpu_device_group_is_finalized(group))
return -EINVAL;
if (group->ext_mailbox)
return -EEXIST;
if (!ext_mailbox_req)
return -EINVAL;
ret = edgetpu_mailbox_validate_attr(&ext_mailbox_req->attr);
if (ret)
return ret;
attr = ext_mailbox_req->attr;
if (!edgetpu_mailbox_external_check_range(mgr, ext_mailbox_req->start,
ext_mailbox_req->end))
return -ERANGE;
ext_mailbox = kzalloc(sizeof(*ext_mailbox), GFP_KERNEL);
if (!ext_mailbox)
return -ENOMEM;
bmap = ext_mailbox_req->mbox_map;
count = __sw_hweight32(bmap);
ext_mailbox->descriptors =
kcalloc(count, sizeof(struct edgetpu_mailbox_descriptor), GFP_KERNEL);
if (!ext_mailbox->descriptors) {
kfree(ext_mailbox);
return -ENOMEM;
}
ext_mailbox->attr = attr;
ext_mailbox->etdev = group->etdev;
ext_mailbox->mbox_type = ext_mailbox_req->mbox_type;
start = ext_mailbox_req->start;
end = ext_mailbox_req->end;
write_lock_irqsave(&mgr->mailboxes_lock, flags);
while (bmap) {
i = ffs(bmap) + start - 1;
if (i > end) {
ret = -EINVAL;
goto unlock;
}
if (mgr->mailboxes[i]) {
ret = -EBUSY;
goto unlock;
}
bmap = bmap & (bmap - 1);
}
bmap = ext_mailbox_req->mbox_map;
while (bmap) {
i = ffs(bmap) + start - 1;
mailbox = edgetpu_mailbox_create_locked(mgr, i);
if (!IS_ERR(mailbox)) {
mgr->mailboxes[i] = mailbox;
ext_mailbox->descriptors[j++].mailbox = mailbox;
} else {
ret = PTR_ERR(mailbox);
goto release;
}
bmap = bmap & (bmap - 1);
}
ext_mailbox->count = j;
ret = edgetpu_mailbox_external_alloc_queue_batch(ext_mailbox);
if (ret)
goto release;
write_unlock_irqrestore(&mgr->mailboxes_lock, flags);
for (i = 0; i < count; i++) {
mailbox = ext_mailbox->descriptors[i].mailbox;
mailbox->internal.group = edgetpu_device_group_get(group);
}
group->ext_mailbox = ext_mailbox;
return 0;
release:
while (j--)
edgetpu_mailbox_remove_locked(mgr, ext_mailbox->descriptors[j].mailbox);
unlock:
write_unlock_irqrestore(&mgr->mailboxes_lock, flags);
kfree(ext_mailbox->descriptors);
kfree(ext_mailbox);
return ret;
}
/* Caller must hold @group->lock. */
static void edgetpu_mailbox_external_free(struct edgetpu_device_group *group)
{
struct edgetpu_mailbox_manager *mgr;
struct edgetpu_mailbox *mailbox;
struct edgetpu_external_mailbox *ext_mailbox;
u32 i;
ext_mailbox = group->ext_mailbox;
if (!ext_mailbox)
return;
mgr = ext_mailbox->etdev->mailbox_manager;
edgetpu_mailbox_external_free_queue_batch(ext_mailbox);
for (i = 0; i < ext_mailbox->count; i++) {
mailbox = ext_mailbox->descriptors[i].mailbox;
edgetpu_device_group_put(mailbox->internal.group);
edgetpu_mailbox_remove(mgr, mailbox);
}
kfree(ext_mailbox->descriptors);
kfree(ext_mailbox);
group->ext_mailbox = NULL;
}
static int edgetpu_mailbox_external_alloc_enable(struct edgetpu_client *client,
struct edgetpu_external_mailbox_req *req)
{
int ret = 0;
struct edgetpu_device_group *group;
mutex_lock(&client->group_lock);
if (!client->group || !edgetpu_device_group_is_leader(client->group, client)) {
mutex_unlock(&client->group_lock);
return -EINVAL;
}
group = edgetpu_device_group_get(client->group);
mutex_unlock(&client->group_lock);
if (edgetpu_pm_get_if_powered(group->etdev->pm)) {
mutex_lock(&group->lock);
ret = edgetpu_mailbox_external_alloc(group, req);
if (ret) {
mutex_unlock(&group->lock);
goto err;
}
edgetpu_mailbox_init_external_mailbox(group->ext_mailbox);
ret = edgetpu_mailbox_activate_external_mailbox(group);
mutex_unlock(&group->lock);
edgetpu_pm_put(group->etdev->pm);
goto out;
} else {
mutex_lock(&group->lock);
ret = edgetpu_mailbox_external_alloc(group, req);
mutex_unlock(&group->lock);
goto out;
}
err:
edgetpu_pm_put(group->etdev->pm);
out:
edgetpu_device_group_put(group);
return ret;
}
static int edgetpu_mailbox_external_disable_free(struct edgetpu_client *client)
{
struct edgetpu_device_group *group;
mutex_lock(&client->group_lock);
if (!client->group || !edgetpu_device_group_is_leader(client->group, client)) {
mutex_unlock(&client->group_lock);
return -EINVAL;
}
group = edgetpu_device_group_get(client->group);
mutex_unlock(&client->group_lock);
if (edgetpu_pm_get_if_powered(group->etdev->pm)) {
mutex_lock(&group->lock);
edgetpu_mailbox_external_disable_free_locked(group);
mutex_unlock(&group->lock);
edgetpu_pm_put(group->etdev->pm);
} else {
mutex_lock(&group->lock);
edgetpu_mailbox_external_free(group);
mutex_unlock(&group->lock);
}
edgetpu_device_group_put(group);
return 0;
}
void edgetpu_mailbox_external_disable_free_locked(struct edgetpu_device_group *group)
{
if (!group->dev_inaccessible) {
/*
* Deactivate only fails if f/w is unresponsive which will put group
* in errored state or mailbox physically disabled before requesting
* deactivate which will never be the case.
*/
edgetpu_mailbox_deactivate_external_mailbox(group);
edgetpu_mailbox_disable_external_mailbox(group);
}
edgetpu_mailbox_external_free(group);
}
static int edgetpu_mailbox_external_enable_by_id(struct edgetpu_client *client, int mailbox_id)
{
int ret;
if (!edgetpu_wakelock_lock(client->wakelock)) {
etdev_err(client->etdev, "Enabling mailbox %d needs wakelock acquired\n",
mailbox_id);
edgetpu_wakelock_unlock(client->wakelock);
return -EAGAIN;
}
etdev_dbg(client->etdev, "Enabling mailbox: %d\n", mailbox_id);
ret = edgetpu_mailbox_activate(client->etdev, mailbox_id, -1, false);
if (ret)
etdev_err(client->etdev, "Activate mailbox %d failed: %d", mailbox_id, ret);
else
edgetpu_wakelock_inc_event_locked(client->wakelock,
EDGETPU_WAKELOCK_EVENT_EXT_MAILBOX);
edgetpu_wakelock_unlock(client->wakelock);
return ret;
}
static int edgetpu_mailbox_external_disable_by_id(struct edgetpu_client *client, int mailbox_id)
{
int ret = 0;
/*
* A successful enable_ext() increases the wakelock event which prevents wakelock being
* released, so theoretically the check fail here can only happen when enable_ext() is
* failed or not called before.
*/
if (!edgetpu_wakelock_lock(client->wakelock)) {
etdev_err(client->etdev, "Disabling mailbox %d needs wakelock acquired\n",
mailbox_id);
edgetpu_wakelock_unlock(client->wakelock);
return -EAGAIN;
}
etdev_dbg(client->etdev, "Disabling mailbox: %d\n", mailbox_id);
edgetpu_mailbox_deactivate(client->etdev, mailbox_id);
edgetpu_wakelock_dec_event_locked(client->wakelock, EDGETPU_WAKELOCK_EVENT_EXT_MAILBOX);
edgetpu_wakelock_unlock(client->wakelock);
return ret;
}
int edgetpu_mailbox_activate_external_mailbox(struct edgetpu_device_group *group)
{
struct edgetpu_external_mailbox *ext_mailbox = group->ext_mailbox;
uint vcid = group->vcid;
u32 mbox_map = 0, i;
int ret;
if (!ext_mailbox)
return -ENOENT;
for (i = 0; i < ext_mailbox->count; i++)
mbox_map |= BIT(ext_mailbox->descriptors[i].mailbox->mailbox_id);
ret = edgetpu_mailbox_activate_bulk(ext_mailbox->etdev, mbox_map, vcid, false);
if (ret)
etdev_err(group->etdev, "Activate mailbox bulk failed: %d", ret);
return ret;
}
void edgetpu_mailbox_disable_external_mailbox(struct edgetpu_device_group *group)
{
u32 i;
struct edgetpu_external_mailbox *ext_mailbox = group->ext_mailbox;
if (!ext_mailbox)
return;
for (i = 0; i < ext_mailbox->count; i++)
edgetpu_mailbox_disable(ext_mailbox->descriptors[i].mailbox);
}
void edgetpu_mailbox_deactivate_external_mailbox(struct edgetpu_device_group *group)
{
u32 i, mbox_map = 0;
struct edgetpu_external_mailbox *ext_mailbox = group->ext_mailbox;
if (!ext_mailbox)
return;
for (i = 0; i < ext_mailbox->count; i++)
mbox_map |= BIT(ext_mailbox->descriptors[i].mailbox->mailbox_id);
etdev_dbg(ext_mailbox->etdev, "Disabling mailboxes in map: %x\n", mbox_map);
edgetpu_mailbox_deactivate_bulk(ext_mailbox->etdev, mbox_map);
}
int edgetpu_mailbox_enable_ext(struct edgetpu_client *client, int mailbox_id,
struct edgetpu_external_mailbox_req *ext_mailbox_req)
{
if (mailbox_id == EDGETPU_MAILBOX_ID_USE_ASSOC)
return edgetpu_mailbox_external_alloc_enable(client, ext_mailbox_req);
else
return edgetpu_mailbox_external_enable_by_id(client, mailbox_id);
}
int edgetpu_mailbox_disable_ext(struct edgetpu_client *client, int mailbox_id)
{
if (mailbox_id == EDGETPU_MAILBOX_ID_USE_ASSOC)
return edgetpu_mailbox_external_disable_free(client);
else
return edgetpu_mailbox_external_disable_by_id(client, mailbox_id);
}
int edgetpu_mailbox_activate_bulk(struct edgetpu_dev *etdev, u32 mailbox_map, s16 vcid,
bool first_open)
{
struct edgetpu_handshake *eh = &etdev->mailbox_manager->open_devices;
int ret = 0;
mutex_lock(&eh->lock);
if (mailbox_map & ~eh->fw_state)
ret = edgetpu_kci_open_device(etdev->kci, mailbox_map & ~eh->fw_state, vcid,
first_open);
if (!ret) {
eh->state |= mailbox_map;
eh->fw_state |= mailbox_map;
}
mutex_unlock(&eh->lock);
/*
* We are observing OPEN_DEVICE KCI fails while other KCIs (usage update / shutdown) still
* succeed and no firmware crash is reported. Kick off the firmware restart when we are
* facing this and hope this can rescue the device from the bad state.
*/
if (ret == -ETIMEDOUT)
edgetpu_watchdog_bite(etdev, false);
return ret;
}
int edgetpu_mailbox_activate(struct edgetpu_dev *etdev, u32 mailbox_id, s16 vcid, bool first_open)
{
return edgetpu_mailbox_activate_bulk(etdev, BIT(mailbox_id), vcid, first_open);
}
void edgetpu_mailbox_deactivate_bulk(struct edgetpu_dev *etdev, u32 mailbox_map)
{
struct edgetpu_handshake *eh = &etdev->mailbox_manager->open_devices;
int ret = 0;
mutex_lock(&eh->lock);
if (mailbox_map & eh->fw_state)
ret = edgetpu_kci_close_device(etdev->kci, mailbox_map & eh->fw_state);
if (ret)
etdev_err(etdev, "Deactivate mailbox for map %x failed: %d", mailbox_map, ret);
/*
* Always clears the states, FW should never reject CLOSE_DEVICE requests unless it's
* unresponsive.
*/
eh->state &= ~mailbox_map;
eh->fw_state &= ~mailbox_map;
mutex_unlock(&eh->lock);
}
void edgetpu_mailbox_deactivate(struct edgetpu_dev *etdev, u32 mailbox_id)
{
edgetpu_mailbox_deactivate_bulk(etdev, BIT(mailbox_id));
}
void edgetpu_handshake_clear_fw_state(struct edgetpu_handshake *eh)
{
mutex_lock(&eh->lock);
eh->fw_state = 0;
mutex_unlock(&eh->lock);
}