blob: 115341dba4bb3cca8c5756bfeb586275383cf8ce [file] [log] [blame]
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Utility functions of mailbox protocol for Edge TPU ML accelerator.
*
* Copyright (C) 2019 Google, Inc.
*/
#ifndef __EDGETPU_MAILBOX_H__
#define __EDGETPU_MAILBOX_H__
#include <linux/compiler.h>
#include <linux/irqreturn.h>
#include <linux/spinlock.h>
#include <linux/types.h>
#include "edgetpu-internal.h"
#include "edgetpu.h"
#define CIRCULAR_QUEUE_WRAP_BIT (1 << 10)
#define CIRCULAR_QUEUE_INDEX_MASK (CIRCULAR_QUEUE_WRAP_BIT - 1)
#define CIRCULAR_QUEUE_VALID_MASK \
(CIRCULAR_QUEUE_INDEX_MASK | CIRCULAR_QUEUE_WRAP_BIT)
#define CIRCULAR_QUEUE_WRAPPED(idx) ((idx) & CIRCULAR_QUEUE_WRAP_BIT)
#define CIRCULAR_QUEUE_REAL_INDEX(idx) ((idx) & CIRCULAR_QUEUE_INDEX_MASK)
/* CMD_QUEUE_SIZE register is 10 bits wide */
#define MAX_QUEUE_SIZE (CIRCULAR_QUEUE_WRAP_BIT - 1)
/* the index of mailbox for kernel control interface */
#define KERNEL_MAILBOX_INDEX 0
/* Size of CSRs start from cmd_queue_csr_base can be mmap-ed to userspace. */
#define USERSPACE_CSR_SIZE 0x1000ul
/* Mailbox ID to indicate external mailboxes */
#define EDGETPU_MAILBOX_ID_USE_ASSOC -1
struct edgetpu_device_group;
struct edgetpu_mailbox {
uint mailbox_id;
struct edgetpu_dev *etdev;
/* base offset for CSRs in struct edgetpu_mailbox_context_csr */
u32 context_csr_base;
/* base offset for CSRs in struct edgetpu_mailbox_cmd_queue_csr */
u32 cmd_queue_csr_base;
/* base offset for CSRs in struct edgetpu_mailbox_resp_queue_csr */
u32 resp_queue_csr_base;
/*
* Queue-related fields, all of them are in units of number of elements.
*/
u32 cmd_queue_size; /* size of cmd queue */
u32 cmd_queue_tail; /* offset within the cmd queue */
u32 resp_queue_size; /* size of resp queue */
u32 resp_queue_head; /* offset within the resp queue */
/* IRQ handler */
void (*handle_irq)(struct edgetpu_mailbox *mailbox);
/*
* Internal data. It's edgetpu_kci* for KCI mailbox,
* edgetpu_device_group* for VII mailboxes.
*/
union {
struct edgetpu_kci *kci;
struct edgetpu_device_group *group;
} internal;
};
typedef struct edgetpu_coherent_mem edgetpu_queue_mem;
struct edgetpu_vii {
/*
* The mailbox this VII uses, can be NULL when uninitialized or mailbox
* detached.
*/
struct edgetpu_mailbox *mailbox;
struct edgetpu_dev *etdev;
edgetpu_queue_mem cmd_queue_mem;
edgetpu_queue_mem resp_queue_mem;
};
/* Structure to hold info about mailbox and its queues. */
struct edgetpu_mailbox_descriptor {
struct edgetpu_mailbox *mailbox;
edgetpu_queue_mem cmd_queue_mem;
edgetpu_queue_mem resp_queue_mem;
};
enum edgetpu_ext_mailbox_type {
EDGETPU_EXTERNAL_MAILBOX_TYPE_DSP,
EDGETPU_EXTERNAL_MAILBOX_TYPE_AOC,
};
/* Structure to hold multiple external mailboxes allocated for a device group. */
struct edgetpu_external_mailbox {
/* Number of external mailboxes allocated for a device group. */
u32 count;
/* Type of external mailbox */
enum edgetpu_ext_mailbox_type mbox_type;
/* Leader of device group. */
struct edgetpu_dev *etdev;
/* Array of external mailboxes info with length @count. */
struct edgetpu_mailbox_descriptor *descriptors;
/* Mailbox attribute for allocated external mailboxes. */
struct edgetpu_mailbox_attr attr;
};
/* Structure used for requesting to allocate external mailboxes. */
struct edgetpu_external_mailbox_req {
uint start; /* starting index of external mailbox in mailbox_manager */
uint end; /* end index of external mailbox in mailbox_manager */
uint mbox_map; /* bitmap of mailbox indexes to be allocated */
enum edgetpu_ext_mailbox_type mbox_type; /* Type of external mailbox */
struct edgetpu_mailbox_attr attr; /* mailbox attribute for allocation */
};
/*
* Structure for recording the driver state vs FW state.
*
* Example usage:
* @state is a bit mask that denotes each mailbox to which an "OPEN_DEVICE"
* KCI has been sent.
* @fw_state is the bit mask of mailbox IDs for which the FW has received the
* "OPEN_DEVICE" KCI.
* In usual cases @state always equals @fw_state. But when the FW is reloaded,
* @fw_state is reset to zero, then this structure can be used to know the FW
* state is out-of-sync and need further actions.
*/
struct edgetpu_handshake {
struct mutex lock;
/* fields protected by @lock */
u32 state;
u32 fw_state;
};
typedef u32 (*get_csr_base_t)(uint index);
struct edgetpu_mailbox_manager {
struct edgetpu_dev *etdev;
/* total number of mailboxes that edgetpu device could provide */
u8 num_mailbox;
/* indices reserved for VII, the range is [from, to) */
u8 vii_index_from, vii_index_to;
/* indices reserved for P2P, the range is [from, to) */
u8 p2p_index_from, p2p_index_to;
/* indices reserved for external mailboxes */
u8 ext_index_from, ext_index_to;
rwlock_t mailboxes_lock; /* protects mailboxes */
struct edgetpu_mailbox **mailboxes;
/* converts index (0 ~ num_mailbox - 1) of mailbox to CSR offset */
get_csr_base_t get_context_csr_base;
get_csr_base_t get_cmd_queue_csr_base;
get_csr_base_t get_resp_queue_csr_base;
struct edgetpu_handshake open_devices;
};
/* the structure to configure a mailbox manager */
struct edgetpu_mailbox_manager_desc {
u8 num_mailbox;
u8 num_vii_mailbox;
u8 num_p2p_mailbox;
u8 num_ext_mailbox;
get_csr_base_t get_context_csr_base;
get_csr_base_t get_cmd_queue_csr_base;
get_csr_base_t get_resp_queue_csr_base;
};
/* Mailbox CSRs. The order and size are exactly the same as RTL defined. */
/* CSRs that can be accessed by AP kernel only, don't mmap them to userspace */
struct edgetpu_mailbox_context_csr {
u32 context_enable;
u32 priority;
u32 cmd_queue_doorbell_enable;
/* doorbell will be triggered automatically on cmd_queue_tail updated */
u32 cmd_queue_tail_doorbell_enable;
u32 cmd_queue_doorbell_clear;
u32 cmd_queue_address_low;
u32 cmd_queue_address_high;
u32 cmd_queue_size;
u32 resp_queue_doorbell_enable;
u32 resp_queue_tail_doorbell_enable;
u32 resp_queue_address_low;
u32 resp_queue_address_high;
u32 resp_queue_size;
u32 config_spare_0;
u32 config_spare_1;
u32 config_spare_2;
u32 config_spare_3;
} __packed;
/* CSRs that can be accessed by AP runtime */
struct edgetpu_mailbox_cmd_queue_csr {
u32 doorbell_set;
u32 doorbell_status;
u32 head;
u32 tail;
u32 config;
u32 error_status;
} __packed;
struct edgetpu_mailbox_resp_queue_csr {
u32 doorbell_set;
u32 doorbell_clear;
u32 doorbell_status;
u32 head;
u32 tail;
u32 config;
u32 error_status;
} __packed;
/* Mailbox APIs */
/* to specify the operation is toward cmd or resp queue */
enum mailbox_queue_type {
MAILBOX_CMD_QUEUE,
MAILBOX_RESP_QUEUE
};
/*
* Allocates the mailbox manager.
*
* Allocations are device-managed so no release function is needed to free the
* manager.
*/
struct edgetpu_mailbox_manager *
edgetpu_mailbox_create_mgr(struct edgetpu_dev *etdev,
const struct edgetpu_mailbox_manager_desc *desc);
/* interrupt handler */
irqreturn_t edgetpu_mailbox_handle_irq(struct edgetpu_mailbox_manager *mgr);
/*
* Removes the mailbox previously requested from a mailbox manager.
*
* This function doesn't change the state of mailbox enable/disable.
*/
int edgetpu_mailbox_remove(struct edgetpu_mailbox_manager *mgr, struct edgetpu_mailbox *mailbox);
/* Removes and disables all the mailboxes previously requested. */
void edgetpu_mailbox_remove_all(struct edgetpu_mailbox_manager *mgr);
/* configure mailbox */
/* set cmd/resp_queue's address and size */
int edgetpu_mailbox_set_queue(struct edgetpu_mailbox *mailbox,
enum mailbox_queue_type type, u64 addr, u32 size);
void edgetpu_mailbox_set_priority(struct edgetpu_mailbox *mailbox,
u32 priority);
/* Reset mailbox queues, clear out any commands/responses left from before. */
void edgetpu_mailbox_reset(struct edgetpu_mailbox *mailbox);
/*
* Clears any stale doorbell requests and enables the doorbell interrupts
* at the mailbox level
*/
void edgetpu_mailbox_init_doorbells(struct edgetpu_mailbox *mailbox);
/* utility functions for KCI */
/* requests the mailbox for KCI */
struct edgetpu_mailbox *edgetpu_mailbox_kci(
struct edgetpu_mailbox_manager *mgr);
void edgetpu_mailbox_inc_cmd_queue_tail(struct edgetpu_mailbox *mailbox,
u32 inc);
void edgetpu_mailbox_inc_resp_queue_head(struct edgetpu_mailbox *mailbox,
u32 inc);
/* utility functions for VII */
/*
* Request the mailbox with mailbox_id equals @id.
* @id = 0 means there is no preference, @mgr will return a spare mailbox.
*
* Caller calls edgetpu_mailbox_enable() to enable the returned mailbox.
*
* -EBUSY is returned if the requested @id is used or there is no mailbox
* available.
*/
struct edgetpu_mailbox *
edgetpu_mailbox_vii_add(struct edgetpu_mailbox_manager *mgr, uint id);
/*
* Validates the mailbox attributes.
* Returns 0 if valid, otherwise a negative errno.
*
* See the error cases of EDGETPU_CREATE_GROUP in edgetpu.h for when will @attr
* be considered as invalid.
*/
int edgetpu_mailbox_validate_attr(const struct edgetpu_mailbox_attr *attr);
/*
* Sets mailbox and allocates queues to @vii.
*
* @group is the device group that @vii will be associated with,
* @group->mbox_attr is used to set the VII mailbox attributes.
*
* @group->mbox_attr must be checked by edgetpu_mailbox_validate_attr() before
* calling this function.
*
* Returns 0 on success.
*/
int edgetpu_mailbox_init_vii(struct edgetpu_vii *vii,
struct edgetpu_device_group *group);
void edgetpu_mailbox_remove_vii(struct edgetpu_vii *vii);
/*
* Reset all mailboxes CSRs to valid values, needed after the device is power
* gated.
*/
void edgetpu_mailbox_reset_mailboxes(struct edgetpu_mailbox_manager *mgr);
/* For VII and P2P mailboxes to allocate/free queue memory */
int edgetpu_mailbox_alloc_queue(struct edgetpu_dev *etdev,
struct edgetpu_mailbox *mailbox, u32 queue_size,
u32 unit, enum mailbox_queue_type type,
edgetpu_queue_mem *mem);
void edgetpu_mailbox_free_queue(struct edgetpu_dev *etdev,
struct edgetpu_mailbox *mailbox,
edgetpu_queue_mem *mem);
/*
* Re-programs the CSRs of queue addresses, context, priority etc. to @group's
* VII mailbox.
*
* Caller holds @group->lock and ensures @group has mailbox attached.
*/
void edgetpu_mailbox_reinit_vii(struct edgetpu_device_group *group);
/*
* Re-configure VII and external mailbox queues which have an active client, re-using
* existing buffers
*/
void edgetpu_mailbox_restore_active_mailbox_queues(struct edgetpu_dev *etdev);
/* utility functions for P2P */
int edgetpu_mailbox_p2p_batch(struct edgetpu_mailbox_manager *mgr, uint n,
uint skip_i, struct edgetpu_mailbox **mailboxes);
/*
* If @mailbox_id is EDGETPU_MAILBOX_ID_USE_ASSOC, use @ext_mailbox_req to
* allocate external mailboxes and activate the allocated mailboxes.
* Otherwise, activate the external mailbox with id @mailbox_id.
*/
int edgetpu_mailbox_enable_ext(struct edgetpu_client *client, int mailbox_id,
struct edgetpu_external_mailbox_req *ext_mailbox_req);
/*
* Notify firmware of an external mailboxes becoming inactive.
*/
int edgetpu_mailbox_disable_ext(struct edgetpu_client *client, int mailbox_id);
/*
* Activates all mailboxes included in @mailbox_map, OPEN_DEVICE KCI will be sent.
*
* Returns what edgetpu_kci_open_device() returned.
* Caller ensures device is powered on.
*/
int edgetpu_mailbox_activate_bulk(struct edgetpu_dev *etdev, u32 mailbox_map, s16 vcid,
bool first_open);
/*
* Activates @mailbox_id, OPEN_DEVICE KCI will be sent.
*
* If @mailbox_id is known to be activated, KCI is not sent and this function
* returns 0.
*
* Returns what edgetpu_kci_open_device() returned.
* Caller ensures device is powered on.
*/
int edgetpu_mailbox_activate(struct edgetpu_dev *etdev, u32 mailbox_id, s16 vcid, bool first_open);
/*
* Similar to edgetpu_mailbox_activate_bulk() but sends CLOSE_DEVICE KCI with the @mailbox_map
* instead.
*/
void edgetpu_mailbox_deactivate_bulk(struct edgetpu_dev *etdev, u32 mailbox_map);
/*
* Similar to edgetpu_mailbox_activate() but sends CLOSE_DEVICE KCI instead.
*/
void edgetpu_mailbox_deactivate(struct edgetpu_dev *etdev, u32 mailbox_id);
/* Sets @eh->fw_state to 0. */
void edgetpu_handshake_clear_fw_state(struct edgetpu_handshake *eh);
/*
* Disables and frees any external mailboxes allocated for @group.
*
* Caller must hold @group->lock and ensure if device is accessible.
*/
void edgetpu_mailbox_external_disable_free_locked(struct edgetpu_device_group *group);
/*
* Re-programs the CSRs of queue addresses, context, priority etc. to @group's
* external mailbox.
*
* Caller holds @group->lock and ensures @group has mailbox attached.
*/
void edgetpu_mailbox_reinit_external_mailbox(struct edgetpu_device_group *group);
/*
* Activates external mailboxes in @group's ext_mailbox.
*
* Caller ensures device is powered and must hold @group->lock.
*/
int edgetpu_mailbox_activate_external_mailbox(struct edgetpu_device_group *group);
/*
* Deactivates external mailboxes in @group's ext_mailbox.
*
* Caller ensures device is powered and must hold @group->lock.
*/
void edgetpu_mailbox_deactivate_external_mailbox(struct edgetpu_device_group *group);
/*
* Disables external mailboxes in @group's ext_mailbox.
*
* Caller ensures device is powered and must hold @group->lock.
*/
void edgetpu_mailbox_disable_external_mailbox(struct edgetpu_device_group *group);
/* Utilities of circular queue operations */
/*
* Returns the number of elements in a circular queue given its @head, @tail,
* and @queue_size.
*/
static inline u32 circular_queue_count(u32 head, u32 tail, u32 queue_size)
{
u32 ret;
if (CIRCULAR_QUEUE_WRAPPED(tail) != CIRCULAR_QUEUE_WRAPPED(head))
ret = queue_size - CIRCULAR_QUEUE_REAL_INDEX(head) +
CIRCULAR_QUEUE_REAL_INDEX(tail);
else
ret = tail - head;
if (unlikely(ret > queue_size))
return 0;
return ret;
}
/* Increases @index of a circular queue by @inc. */
static inline u32 circular_queue_inc(u32 index, u32 inc, u32 queue_size)
{
u32 new_index = CIRCULAR_QUEUE_REAL_INDEX(index) + inc;
if (unlikely(new_index >= queue_size))
return (index + inc - queue_size) ^ CIRCULAR_QUEUE_WRAP_BIT;
else
return index + inc;
}
/* Macros for accessing mailbox CSRs. */
/* Read mailbox register with no memory barrier / access ordering guarantee. */
#define EDGETPU_MAILBOX_READ(mailbox, base, type, field) \
edgetpu_dev_read_32(mailbox->etdev, base + offsetof(type, field))
/*
* Read mailbox register with memory barrier, ensuring the register read
* completes prior to any following CPU reads by this thread.
*/
#define EDGETPU_MAILBOX_READ_SYNC(mailbox, base, type, field) \
edgetpu_dev_read_32_sync(mailbox->etdev, base + offsetof(type, field))
#define EDGETPU_MAILBOX_CONTEXT_READ(mailbox, field) \
EDGETPU_MAILBOX_READ(mailbox, mailbox->context_csr_base, \
struct edgetpu_mailbox_context_csr, field)
#define EDGETPU_MAILBOX_CMD_QUEUE_READ(mailbox, field) \
EDGETPU_MAILBOX_READ(mailbox, mailbox->cmd_queue_csr_base, \
struct edgetpu_mailbox_cmd_queue_csr, field)
/* Read response queue register, no memory barrier / access ordering. */
#define EDGETPU_MAILBOX_RESP_QUEUE_READ(mailbox, field) \
EDGETPU_MAILBOX_READ(mailbox, mailbox->resp_queue_csr_base, \
struct edgetpu_mailbox_resp_queue_csr, field)
/* Read response queue register with memory barrier. */
#define EDGETPU_MAILBOX_RESP_QUEUE_READ_SYNC(mailbox, field) \
EDGETPU_MAILBOX_READ_SYNC(mailbox, mailbox->resp_queue_csr_base, \
struct edgetpu_mailbox_resp_queue_csr, field)
/* Write mailbox register with no memory barrier / access ordering guarantee. */
#define EDGETPU_MAILBOX_WRITE(mailbox, base, type, field, value) \
edgetpu_dev_write_32(mailbox->etdev, base + offsetof(type, field), \
value)
/*
* Write mailbox register with memory barrier, ensuring all CPU memory writes
* by this thread complete prior to the register write.
*/
#define EDGETPU_MAILBOX_WRITE_SYNC(mailbox, base, type, field, value) \
edgetpu_dev_write_32_sync(mailbox->etdev, \
base + offsetof(type, field), \
value)
/* Write context register with no memory barrier / access ordering. */
#define EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, field, value) \
EDGETPU_MAILBOX_WRITE(mailbox, mailbox->context_csr_base, \
struct edgetpu_mailbox_context_csr, field, value)
/* Write context register with memory barrier. */
#define EDGETPU_MAILBOX_CONTEXT_WRITE_SYNC(mailbox, field, value) \
EDGETPU_MAILBOX_WRITE_SYNC(mailbox, mailbox->context_csr_base, \
struct edgetpu_mailbox_context_csr, field, \
value)
/* Write command queue register with no memory barrier / access ordering. */
#define EDGETPU_MAILBOX_CMD_QUEUE_WRITE(mailbox, field, value) \
EDGETPU_MAILBOX_WRITE(mailbox, mailbox->cmd_queue_csr_base, \
struct edgetpu_mailbox_cmd_queue_csr, field, \
value)
/* Write command queue register with memory barrier. */
#define EDGETPU_MAILBOX_CMD_QUEUE_WRITE_SYNC(mailbox, field, value) \
EDGETPU_MAILBOX_WRITE_SYNC(mailbox, mailbox->cmd_queue_csr_base, \
struct edgetpu_mailbox_cmd_queue_csr, \
field, value)
#define EDGETPU_MAILBOX_RESP_QUEUE_WRITE(mailbox, field, value) \
EDGETPU_MAILBOX_WRITE(mailbox, mailbox->resp_queue_csr_base, \
struct edgetpu_mailbox_resp_queue_csr, field, \
value)
/* Enables a mailbox by setting CSR. */
static inline void edgetpu_mailbox_enable(struct edgetpu_mailbox *mailbox)
{
EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, context_enable, 1);
}
/* Disables a mailbox by setting CSR. */
static inline void edgetpu_mailbox_disable(struct edgetpu_mailbox *mailbox)
{
EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, context_enable, 0);
}
#endif /* __EDGETPU_MAILBOX_H__ */