| /* SPDX-License-Identifier: GPL-2.0 */ |
| /* |
| * Kernel Control Interface, implements the protocol between AP kernel and TPU |
| * firmware. |
| * |
| * Copyright (C) 2019 Google, Inc. |
| */ |
| #ifndef __EDGETPU_KCI_H__ |
| #define __EDGETPU_KCI_H__ |
| |
| #include <linux/dma-direction.h> |
| #include <linux/list.h> |
| #include <linux/mutex.h> |
| #include <linux/seq_file.h> |
| #include <linux/spinlock.h> |
| #include <linux/types.h> |
| #include <linux/wait.h> |
| |
| #include "edgetpu-firmware.h" |
| #include "edgetpu-internal.h" |
| #include "edgetpu-mailbox.h" |
| |
| /* |
| * Maximum number of outstanding KCI requests from firmware |
| * This is used to size a circular buffer, so it must be a power of 2 |
| */ |
| #define REVERSE_KCI_BUFFER_SIZE (32) |
| |
| /* |
| * The status field in a firmware response is set to this by us when the |
| * response is fetched from the queue. |
| */ |
| #define KCI_STATUS_OK (0) |
| /* |
| * edgetpu_kci#wait_list uses this value to record the status of responses |
| * that haven't been received yet. |
| */ |
| #define KCI_STATUS_WAITING_RESPONSE (1) |
| /* |
| * Used when an expected response is not received, see the documentation of |
| * edgetpu_kci_consume_wait_list() for details. |
| * |
| */ |
| #define KCI_STATUS_NO_RESPONSE (2) |
| |
| |
| /* |
| * Command/response sequence numbers capped at half the range of the 64-bit |
| * value range. The second half is reserved for incoming requests from firmware. |
| * These are tagged with the MSB set. |
| */ |
| #define KCI_REVERSE_FLAG (0x8000000000000000ull) |
| |
| /* command/response queue elements for KCI */ |
| |
| struct edgetpu_dma_descriptor { |
| u64 address; |
| u32 size; |
| u32 flags; |
| }; |
| |
| struct edgetpu_command_element { |
| /* |
| * Set by edgetpu_kci_push_cmd() in case of KCI cmd and copied from |
| * the RKCI cmd in case of RKCI response. |
| */ |
| u64 seq; |
| u16 code; |
| u16 reserved[3]; /* explicit padding, does not affect alignment */ |
| struct edgetpu_dma_descriptor dma; |
| }; |
| |
| struct edgetpu_kci_response_element { |
| u64 seq; |
| u16 code; |
| /* |
| * Reserved for host use - firmware can't touch this. |
| * If a value is written here it will be discarded and overwritten |
| * during response processing. However, when repurposed as an RKCI |
| * command, the FW can set this field. |
| */ |
| u16 status; |
| /* |
| * Return value is not currently needed by KCI command responses. |
| * For reverse KCI commands this is set as value2. |
| */ |
| u32 retval; |
| } __packed; |
| |
| /* VII response element */ |
| /* The size of this structure must match the runtime definition. */ |
| struct edgetpu_vii_response_element { |
| u64 seq; |
| u16 code; |
| u8 reserved[6]; /* padding */ |
| u64 retval; |
| } __packed; |
| |
| /* |
| * Definition of code in command elements. |
| * Code for KCI is a 16-bit unsigned integer. |
| */ |
| enum edgetpu_kci_code { |
| KCI_CODE_ACK = 0, |
| KCI_CODE_UNMAP_BUFFER = 1, |
| KCI_CODE_MAP_LOG_BUFFER = 2, |
| KCI_CODE_JOIN_GROUP = 3, |
| KCI_CODE_LEAVE_GROUP = 4, |
| KCI_CODE_MAP_TRACE_BUFFER = 5, |
| KCI_CODE_SHUTDOWN = 7, |
| KCI_CODE_GET_DEBUG_DUMP = 8, |
| KCI_CODE_OPEN_DEVICE = 9, |
| KCI_CODE_CLOSE_DEVICE = 10, |
| KCI_CODE_FIRMWARE_INFO = 11, |
| /* TODO(b/271372136): remove v1 when v1 firmware no longer in use. */ |
| KCI_CODE_GET_USAGE_V1 = 12, |
| KCI_CODE_NOTIFY_THROTTLING = 13, |
| KCI_CODE_BLOCK_BUS_SPEED_CONTROL = 14, |
| /* 15..18 not implemented in this branch */ |
| KCI_CODE_FIRMWARE_TRACING_LEVEL = 19, |
| /* 20 not implemented in this branch */ |
| KCI_CODE_GET_USAGE_V2 = 21, |
| |
| KCI_CODE_RKCI_ACK = 256, |
| }; |
| |
| /* |
| * Definition of reverse KCI request code ranges |
| * 16-bit unsigned integer |
| * First half is reserved for chip specific codes, |
| * Generic codes can use the second half. |
| */ |
| enum edgetpu_reverse_kci_code { |
| RKCI_CHIP_CODE_FIRST = 0, |
| RKCI_CHIP_CODE_LAST = 0x7FFF, |
| RKCI_GENERIC_CODE_FIRST = 0x8000, |
| RKCI_FIRMWARE_CRASH = RKCI_GENERIC_CODE_FIRST + 0, |
| RKCI_JOB_LOCKUP = RKCI_GENERIC_CODE_FIRST + 1, |
| RKCI_GENERIC_CODE_LAST = 0xFFFF, |
| }; |
| |
| /* |
| * Definition of code in response elements. |
| * It is a 16-bit unsigned integer. |
| */ |
| enum edgetpu_kci_error { |
| KCI_ERROR_OK = 0, /* Not an error; returned on success */ |
| KCI_ERROR_CANCELLED = 1, |
| KCI_ERROR_UNKNOWN = 2, |
| KCI_ERROR_INVALID_ARGUMENT = 3, |
| KCI_ERROR_DEADLINE_EXCEEDED = 4, |
| KCI_ERROR_NOT_FOUND = 5, |
| KCI_ERROR_ALREADY_EXISTS = 6, |
| KCI_ERROR_PERMISSION_DENIED = 7, |
| KCI_ERROR_RESOURCE_EXHAUSTED = 8, |
| KCI_ERROR_FAILED_PRECONDITION = 9, |
| KCI_ERROR_ABORTED = 10, |
| KCI_ERROR_OUT_OF_RANGE = 11, |
| KCI_ERROR_UNIMPLEMENTED = 12, |
| KCI_ERROR_INTERNAL = 13, |
| KCI_ERROR_UNAVAILABLE = 14, |
| KCI_ERROR_DATA_LOSS = 15, |
| KCI_ERROR_UNAUTHENTICATED = 16, |
| }; |
| |
| struct edgetpu_kci_wait_list { |
| struct list_head list; |
| /* |
| * Its content will be updated once a response with sequence number |
| * equal to resp->seq is received. |
| */ |
| struct edgetpu_kci_response_element *resp; |
| }; |
| |
| /* Struct to hold a circular buffer for incoming KCI responses */ |
| struct edgetpu_reverse_kci { |
| unsigned long head; |
| unsigned long tail; |
| struct edgetpu_kci_response_element buffer[REVERSE_KCI_BUFFER_SIZE]; |
| /* Lock to push elements in the buffer from the interrupt handler */ |
| spinlock_t producer_lock; |
| /* Lock to pop elements from the buffer in the worker */ |
| spinlock_t consumer_lock; |
| /* Worker to handle responses */ |
| struct work_struct work; |
| }; |
| |
| struct edgetpu_kci { |
| struct edgetpu_mailbox *mailbox; |
| struct mutex mailbox_lock; /* protects mailbox */ |
| u64 cur_seq; |
| struct edgetpu_command_element *cmd_queue; |
| struct mutex cmd_queue_lock; /* protects cmd_queue */ |
| /* Command queue buffer */ |
| struct edgetpu_coherent_mem cmd_queue_mem; |
| struct edgetpu_kci_response_element *resp_queue; |
| spinlock_t resp_queue_lock; /* protects resp_queue */ |
| /* Response queue buffer */ |
| struct edgetpu_coherent_mem resp_queue_mem; |
| /* queue for waiting for the response doorbell to be rung */ |
| wait_queue_head_t resp_doorbell_waitq; |
| /* add to this list if a command needs to wait for a response */ |
| struct list_head wait_list; |
| spinlock_t wait_list_lock; /* protects wait_list */ |
| /* queue for waiting for the wait_list to be consumed */ |
| wait_queue_head_t wait_list_waitq; |
| struct work_struct work; /* worker of consuming responses */ |
| /* Handler for reverse (firmware -> kernel) requests */ |
| struct edgetpu_reverse_kci rkci; |
| struct work_struct usage_work; /* worker that sends update usage KCI */ |
| }; |
| |
| struct edgetpu_kci_device_group_detail { |
| u8 n_dies; |
| /* virtual ID from 0 ~ n_dies - 1 */ |
| /* ID 0 for the group master */ |
| u8 vid; |
| u8 reserved[6]; /* padding */ |
| }; |
| |
| struct edgetpu_kci_open_device_detail { |
| /* The bit map of mailboxes to be opened. */ |
| u16 mailbox_map; |
| /* |
| * Virtual context ID @mailbox_id is associated to. |
| * For device groups with @mailbox_detachable attribute the mailbox attached to the group |
| * can be different after wakelock re-acquired. Firmware uses this VCID to identify the |
| * device group. |
| */ |
| u16 vcid; |
| /* |
| * Extra flags for the attributes of this request. |
| * Set RESERVED bits to 0 to ensure backwards compatibility. |
| * |
| * Bitfields: |
| * [0:0] - first_open: Specifies if this is the first time we are calling mailbox open |
| * KCI for this VCID after it has been allocated to a device group. This allows |
| * firmware to clean up/reset the memory allocator for that partition. |
| * [31:1] - RESERVED |
| */ |
| u32 flags; |
| }; |
| |
| /* |
| * Initializes a KCI object. |
| * |
| * Will request a mailbox from @mgr and allocate cmd/resp queues. |
| */ |
| int edgetpu_kci_init(struct edgetpu_mailbox_manager *mgr, |
| struct edgetpu_kci *kci); |
| /* |
| * Re-initializes the initialized KCI object. |
| * |
| * This function is used when the TPU device is reset, it re-programs CSRs |
| * related to KCI mailbox. |
| * |
| * Returns 0 on success, -errno on error. |
| */ |
| int edgetpu_kci_reinit(struct edgetpu_kci *kci); |
| /* |
| * Releases resources allocated by @kci. |
| * |
| * Note: must invoke this function after the interrupt of mailbox disabled and |
| * before free the mailbox pointer. |
| */ |
| void edgetpu_kci_release(struct edgetpu_dev *etdev, struct edgetpu_kci *kci); |
| |
| /* |
| * Pushes an element to cmd queue. |
| * |
| * @cmd's seq field will be set. |
| * Will update the CMD_QUEUE_TAIL CSR. |
| * |
| * @resp will NOT be updated immediately, instead, it will be appended to the |
| * wait_list of @kci. Once the response of @cmd is received, @resp will be |
| * updated. Compare the value of resp->code with KCI_CODE_WAITING_RESPONSE to |
| * check if the response is received. |
| * @resp can be NULL if the command doesn't need a response. |
| * |
| * This is a synchronous function. If the cmd queue is full, it will wait until |
| * the queue is consumed. |
| * |
| * Returns 0 on success, or a negative errno on error. |
| */ |
| int edgetpu_kci_push_cmd(struct edgetpu_kci *kci, |
| struct edgetpu_command_element *cmd, |
| struct edgetpu_kci_response_element *resp); |
| |
| /* |
| * Pushes an element to cmd queue and waits for the response. |
| * Returns -ETIMEDOUT if no response is received within KCI_TIMEOUT. |
| * |
| * Returns the code of response, or a negative errno on error. |
| */ |
| int edgetpu_kci_send_cmd(struct edgetpu_kci *kci, |
| struct edgetpu_command_element *cmd); |
| |
| /* |
| * Sends a FIRMWARE_INFO command and expects a response with a |
| * edgetpu_fw_info struct filled out, including what firmware type is running, |
| * along with build CL and time. |
| * Also serves as an initial handshake with firmware at load time. |
| * |
| * @fw_info: a struct edgetpu_fw_info to be filled out by fw |
| * |
| * Returns >=0 edgetpu_fw_flavor when response received from firmware, |
| * <0 on error communicating with firmware (typically -ETIMEDOUT). |
| */ |
| enum edgetpu_fw_flavor edgetpu_kci_fw_info( |
| struct edgetpu_kci *kci, struct edgetpu_fw_info *fw_info); |
| |
| /* |
| * Schedules a worker to call edgetpu_kci_update_usage(). |
| * |
| * For functions that don't require the usage to be updated immediately, use |
| * this function instead of edgetpu_kci_update_usage(). |
| */ |
| void edgetpu_kci_update_usage_async(struct edgetpu_dev *etdev); |
| |
| /* |
| * Retrieves usage tracking data from firmware, update info on host. |
| * Also used as a watchdog ping to firmware. |
| * |
| * Returns KCI response code on success or < 0 on error (typically -ETIMEDOUT). |
| */ |
| int edgetpu_kci_update_usage(struct edgetpu_dev *etdev); |
| |
| /* |
| * Works the same as edgetpu_kci_update_usage() except the caller of this |
| * function must guarantee the device stays powered up, typically by calling |
| * edgetpu_pm_get() or by calling this function from the power management |
| * functions themselves. |
| * |
| * Returns KCI response code on success or < 0 on error (typically -ETIMEDOUT). |
| */ |
| int edgetpu_kci_update_usage_locked(struct edgetpu_dev *etdev); |
| |
| /* |
| * Sends the "Map Log Buffer" command and waits for remote response. |
| * |
| * Returns the code of response, or a negative errno on error. |
| */ |
| int edgetpu_kci_map_log_buffer(struct edgetpu_kci *kci, tpu_addr_t tpu_addr, |
| u32 size); |
| |
| /* |
| * Sends the "Map Trace Buffer" command and waits for remote response. |
| * |
| * Returns the code of response, or a negative errno on error. |
| */ |
| int edgetpu_kci_map_trace_buffer(struct edgetpu_kci *kci, tpu_addr_t tpu_addr, |
| u32 size); |
| |
| /* |
| * Sent when a group is created with @n_dies dies, and @etdev is the @vid-th |
| * die in this group. |
| * |
| * Returns the code of response, or a negative errno on error. |
| */ |
| int edgetpu_kci_join_group(struct edgetpu_kci *kci, u8 n_dies, u8 vid); |
| /* Informs the TPU to leave the group it currently belongs to. */ |
| int edgetpu_kci_leave_group(struct edgetpu_kci *kci); |
| |
| /* debugfs mappings dump */ |
| void edgetpu_kci_mappings_show(struct edgetpu_dev *etdev, struct seq_file *s); |
| |
| /* Send shutdown request to firmware */ |
| int edgetpu_kci_shutdown(struct edgetpu_kci *kci); |
| |
| /* Request dump of inaccessible segments from firmware. |
| * |
| * @init_buffer flag is used to indicate that the req is only sent to set the dump buffer address |
| * and size in FW. |
| */ |
| int edgetpu_kci_get_debug_dump(struct edgetpu_kci *kci, tpu_addr_t tpu_addr, |
| size_t size, bool init_buffer); |
| |
| /* |
| * Inform the firmware to prepare to serve VII mailboxes included in @mailbox_map. |
| * |
| * You usually shouldn't call this directly - consider using |
| * edgetpu_mailbox_activate() or edgetpu_mailbox_activate_bulk() instead. |
| */ |
| int edgetpu_kci_open_device(struct edgetpu_kci *kci, u32 mailbox_map, s16 vcid, bool first_open); |
| |
| /* |
| * Inform the firmware that the VII mailboxes included in @mailbox_map are closed. |
| * |
| * You usually shouldn't call this directly - consider using |
| * edgetpu_mailbox_deactivate() or edgetpu_mailbox_deactivate_bulk() instead. |
| */ |
| int edgetpu_kci_close_device(struct edgetpu_kci *kci, u32 mailbox_map); |
| |
| /* Cancel work queues or wait until they're done */ |
| void edgetpu_kci_cancel_work_queues(struct edgetpu_kci *kci); |
| |
| /* |
| * Notify the firmware about throttling and the corresponding power level. |
| * The request is sent only if the device is already powered on. |
| * |
| * Returns KCI response code on success or < 0 on error (typically -ETIMEDOUT). |
| */ |
| int edgetpu_kci_notify_throttling(struct edgetpu_dev *etdev, u32 level); |
| |
| /* |
| * Request the firmware to {un}block modulating bus clock speeds |
| * |
| * Used to prevent conflicts when sending a thermal policy request |
| */ |
| int edgetpu_kci_block_bus_speed_control(struct edgetpu_dev *etdev, bool block); |
| |
| /* Set the firmware tracing level. */ |
| int edgetpu_kci_firmware_tracing_level(struct edgetpu_dev *etdev, unsigned long level, |
| unsigned long *active_level); |
| |
| /* |
| * Send an ack to the FW after handling a reverse KCI request. |
| * |
| * The FW may wait for a response from the kernel for an RKCI request so a |
| * response could be sent as an ack. |
| */ |
| int edgetpu_kci_resp_rkci_ack(struct edgetpu_dev *etdev, |
| struct edgetpu_kci_response_element *rkci_cmd); |
| |
| |
| #endif /* __EDGETPU_KCI_H__ */ |