blob: db5548419fb386495d770841c6c9008a90ef3471 [file] [log] [blame]
// SPDX-License-Identifier: GPL-2.0
/*
* File operations for EdgeTPU ML accel chips.
*
* Copyright (C) 2019 Google, Inc.
*/
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/cdev.h>
#include <linux/cred.h>
#include <linux/debugfs.h>
#include <linux/device.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/ioport.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/mutex.h>
#include <linux/of.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/uaccess.h>
#include <linux/uidgid.h>
#include "edgetpu-config.h"
#include "edgetpu-device-group.h"
#include "edgetpu-dmabuf.h"
#include "edgetpu-dram.h"
#include "edgetpu-firmware.h"
#include "edgetpu-internal.h"
#include "edgetpu-kci.h"
#include "edgetpu-mapping.h"
#include "edgetpu-pm.h"
#include "edgetpu-telemetry.h"
#include "edgetpu-wakelock.h"
#include "edgetpu.h"
#ifdef EDGETPU_FEATURE_INTEROP
#include <soc/google/tpu-ext.h>
#endif
#define CREATE_TRACE_POINTS
#include <trace/events/edgetpu.h>
#define DRIVER_VERSION "1.0"
static struct class *edgetpu_class;
static dev_t edgetpu_basedev;
static atomic_t char_minor = ATOMIC_INIT(-1);
static struct dentry *edgetpu_debugfs_dir;
#define LOCK(client) mutex_lock(&client->group_lock)
#define UNLOCK(client) mutex_unlock(&client->group_lock)
/*
* Locks @client->group_lock and checks whether @client is the leader of a group.
* If @client is not the leader of a group, unlocks group_lock and returns false.
* If @client is the leader of a group, returns true with group_lock held.
*/
static inline bool lock_check_group_leader(struct edgetpu_client *client)
{
LOCK(client);
if (!client->group || !edgetpu_device_group_is_leader(client->group, client)) {
UNLOCK(client);
return false;
}
return true;
}
bool is_edgetpu_file(struct file *file)
{
if (edgetpu_is_external_wrapper_class_file(file))
return true;
return file->f_op == &edgetpu_fops;
}
int edgetpu_open(struct edgetpu_dev_iface *etiface, struct file *file)
{
struct edgetpu_client *client;
/* Set client pointer to NULL if error creating client. */
file->private_data = NULL;
client = edgetpu_client_add(etiface);
if (IS_ERR(client))
return PTR_ERR(client);
file->private_data = client;
return 0;
}
static int edgetpu_fs_open(struct inode *inode, struct file *file)
{
struct edgetpu_dev_iface *etiface =
container_of(inode->i_cdev, struct edgetpu_dev_iface, cdev);
return edgetpu_open(etiface, file);
}
static int edgetpu_fs_release(struct inode *inode, struct file *file)
{
struct edgetpu_client *client = file->private_data;
struct edgetpu_dev *etdev;
uint wakelock_count;
if (!client)
return 0;
etdev = client->etdev;
LOCK(client);
/*
* Safe to read wakelock->req_count here since req_count is only
* modified during [acquire/release]_wakelock ioctl calls which cannot
* race with releasing client/fd.
*/
wakelock_count = NO_WAKELOCK(client->wakelock) ? 1 : client->wakelock->req_count;
/*
* @wakelock_count = 0 means the device might be powered off. Mailbox(EXT/VII) is removed
* when the group is released, So we need to ensure the device should not accessed to
* prevent kernel panic on programming mailbox CSRs.
*/
if (!wakelock_count && client->group)
client->group->dev_inaccessible = true;
UNLOCK(client);
edgetpu_client_remove(client);
/* count was zero if client previously released its wake lock */
if (wakelock_count)
edgetpu_pm_put(etdev->pm);
return 0;
}
static int edgetpu_ioctl_set_eventfd(struct edgetpu_client *client,
struct edgetpu_event_register __user *argp)
{
int ret;
struct edgetpu_event_register eventreg;
if (copy_from_user(&eventreg, argp, sizeof(eventreg)))
return -EFAULT;
if (!lock_check_group_leader(client))
return -EINVAL;
ret = edgetpu_group_set_eventfd(client->group, eventreg.event_id, eventreg.eventfd);
UNLOCK(client);
return ret;
}
static int edgetpu_ioctl_unset_eventfd(struct edgetpu_client *client,
uint event_id)
{
if (!lock_check_group_leader(client))
return -EINVAL;
edgetpu_group_unset_eventfd(client->group, event_id);
UNLOCK(client);
return 0;
}
static int
edgetpu_ioctl_set_perdie_eventfd(struct edgetpu_client *client,
struct edgetpu_event_register __user *argp)
{
struct edgetpu_dev *etdev = client->etdev;
struct edgetpu_event_register eventreg;
if (copy_from_user(&eventreg, argp, sizeof(eventreg)))
return -EFAULT;
if (perdie_event_id_to_num(eventreg.event_id) >=
EDGETPU_NUM_PERDIE_EVENTS)
return -EINVAL;
client->perdie_events |= 1 << perdie_event_id_to_num(eventreg.event_id);
switch (eventreg.event_id) {
case EDGETPU_PERDIE_EVENT_LOGS_AVAILABLE:
return edgetpu_telemetry_set_event(etdev, EDGETPU_TELEMETRY_LOG,
eventreg.eventfd);
case EDGETPU_PERDIE_EVENT_TRACES_AVAILABLE:
return edgetpu_telemetry_set_event(
etdev, EDGETPU_TELEMETRY_TRACE, eventreg.eventfd);
default:
return -EINVAL;
}
}
static int edgetpu_ioctl_unset_perdie_eventfd(struct edgetpu_client *client,
uint event_id)
{
struct edgetpu_dev *etdev = client->etdev;
if (perdie_event_id_to_num(event_id) >= EDGETPU_NUM_PERDIE_EVENTS)
return -EINVAL;
client->perdie_events &= ~(1 << perdie_event_id_to_num(event_id));
switch (event_id) {
case EDGETPU_PERDIE_EVENT_LOGS_AVAILABLE:
edgetpu_telemetry_unset_event(etdev, EDGETPU_TELEMETRY_LOG);
break;
case EDGETPU_PERDIE_EVENT_TRACES_AVAILABLE:
edgetpu_telemetry_unset_event(etdev, EDGETPU_TELEMETRY_TRACE);
break;
default:
return -EINVAL;
}
return 0;
}
static int edgetpu_ioctl_finalize_group(struct edgetpu_client *client)
{
struct edgetpu_device_group *group;
int ret = -EINVAL;
LOCK(client);
group = client->group;
if (!group || !edgetpu_device_group_is_leader(group, client))
goto out_unlock;
/* Finalization has to be performed with device on. */
ret = edgetpu_pm_get(client->etdev->pm);
if (ret) {
etdev_err(client->etdev, "%s: pm_get failed (%d)",
__func__, ret);
goto out_unlock;
}
/*
* Hold the wakelock since we need to decide whether VII should be
* initialized during finalization.
*/
edgetpu_wakelock_lock(client->wakelock);
ret = edgetpu_device_group_finalize(group);
edgetpu_wakelock_unlock(client->wakelock);
edgetpu_pm_put(client->etdev->pm);
out_unlock:
UNLOCK(client);
return ret;
}
static int edgetpu_ioctl_join_group(struct edgetpu_client *client,
u64 leader_fd)
{
struct fd f = fdget(leader_fd);
struct file *file = f.file;
struct edgetpu_client *leader;
int ret;
struct edgetpu_device_group *group;
if (!file) {
ret = -EBADF;
goto out;
}
if (!is_edgetpu_file(file)) {
ret = -EINVAL;
goto out;
}
leader = file->private_data;
if (!leader) {
ret = -EINVAL;
goto out;
}
mutex_lock(&leader->group_lock);
if (!leader->group ||
!edgetpu_device_group_is_leader(leader->group, leader)) {
ret = -EINVAL;
mutex_unlock(&leader->group_lock);
goto out;
}
group = edgetpu_device_group_get(leader->group);
mutex_unlock(&leader->group_lock);
ret = edgetpu_device_group_add(group, client);
edgetpu_device_group_put(group);
out:
fdput(f);
return ret;
}
static int edgetpu_ioctl_create_group(struct edgetpu_client *client,
struct edgetpu_mailbox_attr __user *argp)
{
struct edgetpu_mailbox_attr attr;
struct edgetpu_device_group *group;
if (copy_from_user(&attr, argp, sizeof(attr)))
return -EFAULT;
group = edgetpu_device_group_alloc(client, &attr);
if (IS_ERR(group))
return PTR_ERR(group);
edgetpu_device_group_put(group);
return 0;
}
static int edgetpu_ioctl_map_buffer(struct edgetpu_client *client,
struct edgetpu_map_ioctl __user *argp)
{
struct edgetpu_device_group *group;
struct edgetpu_map_ioctl ibuf;
int ret;
if (copy_from_user(&ibuf, argp, sizeof(ibuf)))
return -EFAULT;
trace_edgetpu_map_buffer_start(&ibuf);
if (!lock_check_group_leader(client))
return -EINVAL;
/* to prevent group being released when we perform map/unmap later */
group = edgetpu_device_group_get(client->group);
/*
* Don't hold @client->group_lock on purpose since
* 1. We don't care whether @client still belongs to @group.
* 2. get_user_pages_fast called by edgetpu_device_group_map() will hold
* mm->mmap_sem, we need to prevent our locks being held around it.
*/
UNLOCK(client);
ret = edgetpu_device_group_map(group, &ibuf);
if (ret)
goto out;
if (copy_to_user(argp, &ibuf, sizeof(ibuf))) {
edgetpu_device_group_unmap(group, ibuf.die_index,
ibuf.device_address,
EDGETPU_MAP_SKIP_CPU_SYNC);
ret = -EFAULT;
}
out:
edgetpu_device_group_put(group);
trace_edgetpu_map_buffer_end(&ibuf);
return ret;
}
static int edgetpu_ioctl_unmap_buffer(struct edgetpu_client *client,
struct edgetpu_map_ioctl __user *argp)
{
struct edgetpu_map_ioctl ibuf;
int ret;
if (copy_from_user(&ibuf, argp, sizeof(ibuf)))
return -EFAULT;
if (!lock_check_group_leader(client))
return -EINVAL;
ret = edgetpu_device_group_unmap(client->group, ibuf.die_index, ibuf.device_address,
ibuf.flags);
UNLOCK(client);
return ret;
}
static int
edgetpu_ioctl_allocate_device_buffer(struct edgetpu_client *client, u64 size)
{
#ifndef EDGETPU_HAS_DEVICE_DRAM
return -ENOTTY;
#else
return edgetpu_device_dram_getfd(client, size);
#endif /* EDGETPU_HAS_DEVICE_DRAM */
}
static int edgetpu_ioctl_sync_buffer(struct edgetpu_client *client,
struct edgetpu_sync_ioctl __user *argp)
{
int ret;
struct edgetpu_sync_ioctl ibuf;
if (copy_from_user(&ibuf, argp, sizeof(ibuf)))
return -EFAULT;
if (!lock_check_group_leader(client))
return -EINVAL;
ret = edgetpu_device_group_sync_buffer(client->group, &ibuf);
UNLOCK(client);
return ret;
}
static int
edgetpu_ioctl_map_dmabuf(struct edgetpu_client *client,
struct edgetpu_map_dmabuf_ioctl __user *argp)
{
struct edgetpu_device_group *group;
struct edgetpu_map_dmabuf_ioctl ibuf;
int ret;
if (copy_from_user(&ibuf, argp, sizeof(ibuf)))
return -EFAULT;
trace_edgetpu_map_dmabuf_start(&ibuf);
if (!lock_check_group_leader(client))
return -EINVAL;
/* to prevent group being released when we perform unmap on fault */
group = edgetpu_device_group_get(client->group);
ret = edgetpu_map_dmabuf(group, &ibuf);
UNLOCK(client);
if (ret)
goto out;
if (copy_to_user(argp, &ibuf, sizeof(ibuf))) {
edgetpu_unmap_dmabuf(group, ibuf.die_index,
ibuf.device_address);
ret = -EFAULT;
}
out:
edgetpu_device_group_put(group);
trace_edgetpu_map_dmabuf_end(&ibuf);
return ret;
}
static int
edgetpu_ioctl_unmap_dmabuf(struct edgetpu_client *client,
struct edgetpu_map_dmabuf_ioctl __user *argp)
{
int ret;
struct edgetpu_map_dmabuf_ioctl ibuf;
if (copy_from_user(&ibuf, argp, sizeof(ibuf)))
return -EFAULT;
if (!lock_check_group_leader(client))
return -EINVAL;
ret = edgetpu_unmap_dmabuf(client->group, ibuf.die_index, ibuf.device_address);
UNLOCK(client);
return ret;
}
static int edgetpu_ioctl_sync_fence_create(
struct edgetpu_client *client,
struct edgetpu_create_sync_fence_data __user *datap)
{
struct edgetpu_create_sync_fence_data data;
int ret;
if (copy_from_user(&data, (void __user *)datap, sizeof(data)))
return -EFAULT;
LOCK(client);
if (!client->group) {
etdev_err(client->etdev, "client creating sync fence not joined to a device group");
UNLOCK(client);
return -EINVAL;
}
ret = edgetpu_sync_fence_create(client->group, &data);
UNLOCK(client);
if (ret)
return ret;
if (copy_to_user((void __user *)datap, &data, sizeof(data)))
ret = -EFAULT;
return ret;
}
static int edgetpu_ioctl_sync_fence_signal(
struct edgetpu_signal_sync_fence_data __user *datap)
{
struct edgetpu_signal_sync_fence_data data;
if (copy_from_user(&data, (void __user *)datap, sizeof(data)))
return -EFAULT;
return edgetpu_sync_fence_signal(&data);
}
static int
edgetpu_ioctl_map_bulk_dmabuf(struct edgetpu_client *client,
struct edgetpu_map_bulk_dmabuf_ioctl __user *argp)
{
struct edgetpu_device_group *group;
struct edgetpu_map_bulk_dmabuf_ioctl ibuf;
int ret;
if (copy_from_user(&ibuf, argp, sizeof(ibuf)))
return -EFAULT;
if (!lock_check_group_leader(client))
return -EINVAL;
/* to prevent group being released when we perform unmap on fault */
group = edgetpu_device_group_get(client->group);
ret = edgetpu_map_bulk_dmabuf(group, &ibuf);
UNLOCK(client);
if (ret)
goto out;
if (copy_to_user(argp, &ibuf, sizeof(ibuf))) {
edgetpu_unmap_bulk_dmabuf(group, ibuf.device_address);
ret = -EFAULT;
}
out:
edgetpu_device_group_put(group);
return ret;
}
static int edgetpu_ioctl_unmap_bulk_dmabuf(
struct edgetpu_client *client,
struct edgetpu_map_bulk_dmabuf_ioctl __user *argp)
{
int ret;
struct edgetpu_map_bulk_dmabuf_ioctl ibuf;
if (copy_from_user(&ibuf, argp, sizeof(ibuf)))
return -EFAULT;
if (!lock_check_group_leader(client))
return -EINVAL;
ret = edgetpu_unmap_bulk_dmabuf(client->group, ibuf.device_address);
UNLOCK(client);
return ret;
}
static int edgetpu_ioctl_sync_fence_status(
struct edgetpu_sync_fence_status __user *datap)
{
struct edgetpu_sync_fence_status data;
int ret;
if (copy_from_user(&data, (void __user *)datap, sizeof(data)))
return -EFAULT;
ret = edgetpu_sync_fence_status(&data);
if (ret)
return ret;
if (copy_to_user((void __user *)datap, &data, sizeof(data)))
ret = -EFAULT;
return ret;
}
static int edgetpu_ioctl_fw_version(struct edgetpu_dev *etdev,
struct edgetpu_fw_version __user *argp)
{
if (etdev->fw_version.kci_version == EDGETPU_INVALID_KCI_VERSION)
return -ENODEV;
if (copy_to_user(argp, &etdev->fw_version, sizeof(*argp)))
return -EFAULT;
return 0;
}
static int edgetpu_ioctl_tpu_timestamp(struct edgetpu_client *client,
__u64 __user *argp)
{
u64 timestamp;
int ret = 0;
if (!edgetpu_wakelock_lock(client->wakelock)) {
edgetpu_wakelock_unlock(client->wakelock);
ret = -EAGAIN;
} else {
timestamp = edgetpu_chip_tpu_timestamp(client->etdev);
edgetpu_wakelock_unlock(client->wakelock);
if (copy_to_user(argp, &timestamp, sizeof(*argp)))
ret = -EFAULT;
}
return ret;
}
static bool edgetpu_ioctl_check_permissions(struct file *file, uint cmd)
{
return file->f_mode & FMODE_WRITE;
}
static int edgetpu_ioctl_release_wakelock(struct edgetpu_client *client)
{
int count;
LOCK(client);
edgetpu_wakelock_lock(client->wakelock);
/* when NO_WAKELOCK: count should be 1 so here is a no-op */
count = edgetpu_wakelock_release(client->wakelock);
if (count < 0) {
edgetpu_wakelock_unlock(client->wakelock);
UNLOCK(client);
return count;
}
if (!count) {
if (client->group)
edgetpu_group_close_and_detach_mailbox(client->group);
edgetpu_pm_put(client->etdev->pm);
}
edgetpu_wakelock_unlock(client->wakelock);
UNLOCK(client);
etdev_dbg(client->etdev, "%s: wakelock req count = %u", __func__,
count);
return 0;
}
static int edgetpu_ioctl_acquire_wakelock(struct edgetpu_client *client)
{
int count;
int ret;
struct edgetpu_thermal *thermal = client->etdev->thermal;
LOCK(client);
/*
* Update client PID; the client may have been passed from the
* edgetpu service that originally created it to a new process.
* By the time the client holds TPU wakelocks it will have been
* passed to the new owning process.
*/
client->pid = current->pid;
client->tgid = current->tgid;
if (edgetpu_thermal_is_suspended(thermal)) {
/* TPU is thermal suspended, so fail acquiring wakelock */
ret = -EAGAIN;
etdev_warn_ratelimited(client->etdev,
"wakelock acquire rejected due to device thermal limit exceeded");
goto error_client_unlock;
} else {
ret = edgetpu_pm_get(client->etdev->pm);
if (ret) {
etdev_warn(client->etdev, "%s: pm_get failed (%d)",
__func__, ret);
goto error_client_unlock;
}
}
edgetpu_wakelock_lock(client->wakelock);
/* when NO_WAKELOCK: count should be 1 so here is a no-op */
count = edgetpu_wakelock_acquire(client->wakelock);
if (count < 0) {
ret = count;
goto error_wakelock_unlock;
}
if (!count) {
if (client->group)
ret = edgetpu_group_attach_and_open_mailbox(client->group);
if (ret) {
etdev_warn(client->etdev, "failed to attach mailbox: %d", ret);
edgetpu_wakelock_release(client->wakelock);
goto error_wakelock_unlock;
}
}
error_wakelock_unlock:
edgetpu_wakelock_unlock(client->wakelock);
/* Balance the power up count due to pm_get above.*/
if (ret || count)
edgetpu_pm_put(client->etdev->pm);
error_client_unlock:
UNLOCK(client);
if (ret)
etdev_err(client->etdev, "client pid %d failed to acquire wakelock", client->pid);
else
etdev_dbg(client->etdev, "%s: wakelock req count = %u", __func__, count + 1);
return ret;
}
static int
edgetpu_ioctl_dram_usage(struct edgetpu_dev *etdev,
struct edgetpu_device_dram_usage __user *argp)
{
struct edgetpu_device_dram_usage dram;
dram.allocated = edgetpu_device_dram_used(etdev);
dram.available = edgetpu_device_dram_available(etdev);
if (copy_to_user(argp, &dram, sizeof(*argp)))
return -EFAULT;
return 0;
}
static int
edgetpu_ioctl_acquire_ext_mailbox(struct edgetpu_client *client,
struct edgetpu_ext_mailbox_ioctl __user *argp)
{
struct edgetpu_ext_mailbox_ioctl ext_mailbox;
int ret;
if (copy_from_user(&ext_mailbox, argp, sizeof(ext_mailbox)))
return -EFAULT;
ret = edgetpu_chip_acquire_ext_mailbox(client, &ext_mailbox);
if (ret)
etdev_err(client->etdev, "client pid %d failed to acquire ext mailbox",
client->pid);
return ret;
}
static int
edgetpu_ioctl_release_ext_mailbox(struct edgetpu_client *client,
struct edgetpu_ext_mailbox_ioctl __user *argp)
{
struct edgetpu_ext_mailbox_ioctl ext_mailbox;
if (copy_from_user(&ext_mailbox, argp, sizeof(ext_mailbox)))
return -EFAULT;
return edgetpu_chip_release_ext_mailbox(client, &ext_mailbox);
}
static int edgetpu_ioctl_get_fatal_errors(struct edgetpu_client *client,
__u32 __user *argp)
{
u32 fatal_errors = 0;
int ret = 0;
LOCK(client);
if (client->group)
fatal_errors = edgetpu_group_get_fatal_errors(client->group);
UNLOCK(client);
if (copy_to_user(argp, &fatal_errors, sizeof(fatal_errors)))
ret = -EFAULT;
return ret;
}
#ifdef EDGETPU_FEATURE_INTEROP
static int edgetpu_ioctl_test_external(struct edgetpu_client *client,
struct edgetpu_test_ext_ioctl __user *argp)
{
int ret = 0;
struct edgetpu_test_ext_ioctl test_ext;
struct edgetpu_ext_client_info client_info;
struct edgetpu_ext_mailbox_info *info;
if (!uid_eq(current_euid(), GLOBAL_ROOT_UID))
return -EPERM;
if (copy_from_user(&test_ext, argp, sizeof(test_ext)))
return -EFAULT;
if (hweight32(test_ext.mbox_bmap) != 1)
return -EINVAL;
client_info.attr = (struct edgetpu_mailbox_attr __user *)test_ext.attrs;
client_info.tpu_fd = test_ext.fd;
client_info.mbox_map = test_ext.mbox_bmap;
info = kmalloc(sizeof(*info) + sizeof(struct edgetpu_ext_mailbox_descriptor), GFP_KERNEL);
if (!info)
return -ENOMEM;
ret = edgetpu_ext_driver_cmd(client->etdev->dev, test_ext.client_type, test_ext.cmd,
&client_info, info);
kfree(info);
return ret;
}
#endif /* EDGETPU_FEATURE_INTEROP */
long edgetpu_ioctl(struct file *file, uint cmd, ulong arg)
{
struct edgetpu_client *client = file->private_data;
void __user *argp = (void __user *)arg;
long ret;
if (!client)
return -ENODEV;
if (!edgetpu_ioctl_check_permissions(file, cmd))
return -EPERM;
switch (cmd) {
case EDGETPU_MAP_BUFFER:
ret = edgetpu_ioctl_map_buffer(client, argp);
break;
case EDGETPU_UNMAP_BUFFER:
ret = edgetpu_ioctl_unmap_buffer(client, argp);
break;
case EDGETPU_SET_EVENTFD:
ret = edgetpu_ioctl_set_eventfd(client, argp);
break;
case EDGETPU_CREATE_GROUP:
ret = edgetpu_ioctl_create_group(client, argp);
break;
case EDGETPU_JOIN_GROUP:
ret = edgetpu_ioctl_join_group(client, (u64)argp);
break;
case EDGETPU_FINALIZE_GROUP:
ret = edgetpu_ioctl_finalize_group(client);
break;
case EDGETPU_SET_PERDIE_EVENTFD:
ret = edgetpu_ioctl_set_perdie_eventfd(client, argp);
break;
case EDGETPU_UNSET_EVENT:
ret = edgetpu_ioctl_unset_eventfd(client, arg);
break;
case EDGETPU_UNSET_PERDIE_EVENT:
ret = edgetpu_ioctl_unset_perdie_eventfd(client, arg);
break;
case EDGETPU_SYNC_BUFFER:
ret = edgetpu_ioctl_sync_buffer(client, argp);
break;
case EDGETPU_MAP_DMABUF:
ret = edgetpu_ioctl_map_dmabuf(client, argp);
break;
case EDGETPU_UNMAP_DMABUF:
ret = edgetpu_ioctl_unmap_dmabuf(client, argp);
break;
case EDGETPU_ALLOCATE_DEVICE_BUFFER:
ret = edgetpu_ioctl_allocate_device_buffer(client, (u64)argp);
break;
case EDGETPU_CREATE_SYNC_FENCE:
ret = edgetpu_ioctl_sync_fence_create(client, argp);
break;
case EDGETPU_SIGNAL_SYNC_FENCE:
ret = edgetpu_ioctl_sync_fence_signal(argp);
break;
case EDGETPU_MAP_BULK_DMABUF:
ret = edgetpu_ioctl_map_bulk_dmabuf(client, argp);
break;
case EDGETPU_UNMAP_BULK_DMABUF:
ret = edgetpu_ioctl_unmap_bulk_dmabuf(client, argp);
break;
case EDGETPU_SYNC_FENCE_STATUS:
ret = edgetpu_ioctl_sync_fence_status(argp);
break;
case EDGETPU_RELEASE_WAKE_LOCK:
ret = edgetpu_ioctl_release_wakelock(client);
break;
case EDGETPU_ACQUIRE_WAKE_LOCK:
ret = edgetpu_ioctl_acquire_wakelock(client);
break;
case EDGETPU_FIRMWARE_VERSION:
ret = edgetpu_ioctl_fw_version(client->etdev, argp);
break;
case EDGETPU_GET_TPU_TIMESTAMP:
ret = edgetpu_ioctl_tpu_timestamp(client, argp);
break;
case EDGETPU_GET_DRAM_USAGE:
ret = edgetpu_ioctl_dram_usage(client->etdev, argp);
break;
case EDGETPU_ACQUIRE_EXT_MAILBOX:
ret = edgetpu_ioctl_acquire_ext_mailbox(client, argp);
break;
case EDGETPU_RELEASE_EXT_MAILBOX:
ret = edgetpu_ioctl_release_ext_mailbox(client, argp);
break;
case EDGETPU_GET_FATAL_ERRORS:
ret = edgetpu_ioctl_get_fatal_errors(client, argp);
break;
#ifdef EDGETPU_FEATURE_INTEROP
case EDGETPU_TEST_EXTERNAL:
ret = edgetpu_ioctl_test_external(client, argp);
break;
#endif /* EDGETPU_FEATURE_INTEROP */
default:
return -ENOTTY; /* unknown command */
}
return ret;
}
static long edgetpu_fs_ioctl(struct file *file, uint cmd, ulong arg)
{
return edgetpu_ioctl(file, cmd, arg);
}
/* Map a region of device/coherent memory. */
static int edgetpu_fs_mmap(struct file *file, struct vm_area_struct *vma)
{
struct edgetpu_client *client = file->private_data;
if (!client)
return -ENODEV;
return edgetpu_mmap(client, vma);
}
#ifndef EDGETPU_FEATURE_MOBILE
static struct edgetpu_dumpregs_range common_statusregs_ranges[] = {
{
.firstreg = EDGETPU_REG_AON_RESET,
.lastreg = EDGETPU_REG_AON_FORCE_QUIESCE,
},
{
.firstreg = EDGETPU_REG_USER_HIB_DMA_PAUSED,
.lastreg = EDGETPU_REG_USER_HIB_DMA_PAUSED,
},
{
.firstreg = EDGETPU_REG_USER_HIB_ERROR_STATUS,
.lastreg = EDGETPU_REG_USER_HIB_ERROR_MASK,
},
{
.firstreg = EDGETPU_REG_SC_CURRENTPC,
.lastreg = EDGETPU_REG_SC_DECODEPC,
},
{
.firstreg = EDGETPU_REG_SC_ERROR,
.lastreg = EDGETPU_REG_SC_ERROR_MASK,
},
{
.firstreg = EDGETPU_REG_SC_ERROR_INFO,
.lastreg = EDGETPU_REG_SC_ERROR_INFO,
},
{
.firstreg = EDGETPU_REG_USER_HIB_INSTRQ_TAIL,
.lastreg = EDGETPU_REG_USER_HIB_INSTRQ_INT_STAT,
},
{
.firstreg = EDGETPU_REG_USER_HIB_SC_HOST_INT_STAT,
.lastreg = EDGETPU_REG_USER_HIB_SC_HOST_INT_STAT,
},
{
.firstreg = EDGETPU_REG_USER_HIB_FATALERR_INT_STAT,
.lastreg = EDGETPU_REG_USER_HIB_FATALERR_INT_STAT,
},
{
.firstreg = EDGETPU_REG_USER_HIB_SNAPSHOT,
.lastreg = EDGETPU_REG_USER_HIB_SNAPSHOT,
},
};
static struct edgetpu_dumpregs_range common_tile_statusregs_ranges[] = {
{
.firstreg = EDGETPU_REG_TILECONF1_DEEPSLEEP,
.lastreg = EDGETPU_REG_TILECONF1_DEEPSLEEP,
},
{
.firstreg = EDGETPU_REG_TILECONF1_ERROR_TILE,
.lastreg = EDGETPU_REG_TILECONF1_ERROR_MASK_TILE,
},
{
.firstreg = EDGETPU_REG_TILECONF1_ERROR_INFO_TILE,
.lastreg = EDGETPU_REG_TILECONF1_ERROR_INFO_TILE,
},
};
static void dump_statusregs_ranges(
struct seq_file *s, struct edgetpu_dev *etdev,
struct edgetpu_dumpregs_range *ranges, int nranges)
{
int i;
enum edgetpu_csrs reg;
uint64_t val;
for (i = 0; i < nranges; i++) {
for (reg = ranges[i].firstreg; reg <= ranges[i].lastreg;
reg += sizeof(val)) {
val = edgetpu_dev_read_64(etdev, reg);
seq_printf(s, "%#08x: %#016llx\n", reg, val);
}
}
}
static void dump_mboxes(struct seq_file *s, struct edgetpu_dev *etdev)
{
enum edgetpu_csrs base;
uint32_t val;
int mbox_id;
int n_p2p_mbox_dump = EDGETPU_NUM_P2P_MAILBOXES;
/* Dump VII mailboxes plus P2P (if any) + KCI. */
for (mbox_id = 0, base = EDGETPU_MBOX_BASE;
mbox_id < EDGETPU_NUM_VII_MAILBOXES + n_p2p_mbox_dump + 1;
mbox_id++, base += EDGETPU_MBOX_CSRS_SIZE) {
int offset;
for (offset = 0x0; offset <= 0x40; offset += sizeof(val)) {
val = edgetpu_dev_read_32(etdev, base + offset);
seq_printf(s, "%#08x: %#08x\n", base + offset, val);
}
for (offset = 0x1000; offset <= 0x1014; offset += sizeof(val)) {
val = edgetpu_dev_read_32(etdev, base + offset);
seq_printf(s, "%#08x: %#08x\n", base + offset, val);
}
for (offset = 0x1800; offset <= 0x1818; offset += sizeof(val)) {
val = edgetpu_dev_read_32(etdev, base + offset);
seq_printf(s, "%#08x: %#08x\n", base + offset, val);
}
}
}
static int statusregs_show(struct seq_file *s, void *data)
{
struct edgetpu_dev *etdev = s->private;
int tileid;
dump_statusregs_ranges(s, etdev, common_statusregs_ranges,
ARRAY_SIZE(common_statusregs_ranges));
dump_statusregs_ranges(s, etdev, edgetpu_chip_statusregs_ranges,
edgetpu_chip_statusregs_nranges);
for (tileid = 0; tileid < EDGETPU_NTILES; tileid++) {
edgetpu_dev_write_64(etdev, EDGETPU_REG_USER_HIB_TILECONFIG1,
tileid);
seq_printf(s, "tile %d:\n", tileid);
dump_statusregs_ranges(
s, etdev, common_tile_statusregs_ranges,
ARRAY_SIZE(common_tile_statusregs_ranges));
dump_statusregs_ranges(s, etdev,
edgetpu_chip_tile_statusregs_ranges,
edgetpu_chip_tile_statusregs_nranges);
}
dump_mboxes(s, etdev);
return 0;
}
static int statusregs_open(struct inode *inode, struct file *file)
{
return single_open(file, statusregs_show, inode->i_private);
}
static const struct file_operations statusregs_ops = {
.open = statusregs_open,
.read = seq_read,
.llseek = seq_lseek,
.owner = THIS_MODULE,
.release = single_release,
};
#endif /* EDGETPU_FEATURE_MOBILE */
static int mappings_show(struct seq_file *s, void *data)
{
struct edgetpu_dev *etdev = s->private;
struct edgetpu_list_group *l;
struct edgetpu_device_group *group;
mutex_lock(&etdev->groups_lock);
etdev_for_each_group(etdev, l, group)
edgetpu_group_mappings_show(group, s);
mutex_unlock(&etdev->groups_lock);
edgetpu_kci_mappings_show(etdev, s);
return 0;
}
static int mappings_open(struct inode *inode, struct file *file)
{
return single_open(file, mappings_show, inode->i_private);
}
static const struct file_operations mappings_ops = {
.open = mappings_open,
.read = seq_read,
.llseek = seq_lseek,
.owner = THIS_MODULE,
.release = single_release,
};
static int syncfences_open(struct inode *inode, struct file *file)
{
return single_open(file, edgetpu_sync_fence_debugfs_show, inode->i_private);
}
static const struct file_operations syncfences_ops = {
.open = syncfences_open,
.read = seq_read,
.llseek = seq_lseek,
.owner = THIS_MODULE,
.release = single_release,
};
static int edgetpu_pm_debugfs_set_wakelock(void *data, u64 val)
{
struct edgetpu_dev *etdev = data;
int ret = 0;
if (val)
ret = edgetpu_pm_get(etdev->pm);
else
edgetpu_pm_put(etdev->pm);
return ret;
}
DEFINE_DEBUGFS_ATTRIBUTE(fops_wakelock, NULL, edgetpu_pm_debugfs_set_wakelock,
"%llu\n");
static void edgetpu_fs_setup_debugfs(struct edgetpu_dev *etdev)
{
etdev->d_entry =
debugfs_create_dir(etdev->dev_name, edgetpu_debugfs_dir);
if (IS_ERR_OR_NULL(etdev->d_entry)) {
etdev_warn(etdev, "Failed to setup debugfs\n");
return;
}
debugfs_create_file("mappings", 0440, etdev->d_entry,
etdev, &mappings_ops);
debugfs_create_file("syncfences", 0440, etdev->d_entry, etdev, &syncfences_ops);
debugfs_create_file("wakelock", 0220, etdev->d_entry, etdev,
&fops_wakelock);
#ifndef EDGETPU_FEATURE_MOBILE
debugfs_create_file("statusregs", 0440, etdev->d_entry, etdev,
&statusregs_ops);
#endif
}
static ssize_t firmware_crash_count_show(
struct device *dev, struct device_attribute *attr,
char *buf)
{
struct edgetpu_dev *etdev = dev_get_drvdata(dev);
return scnprintf(buf, PAGE_SIZE, "%u\n", etdev->firmware_crash_count);
}
static DEVICE_ATTR_RO(firmware_crash_count);
static ssize_t watchdog_timeout_count_show(
struct device *dev, struct device_attribute *attr,
char *buf)
{
struct edgetpu_dev *etdev = dev_get_drvdata(dev);
return scnprintf(buf, PAGE_SIZE, "%u\n", etdev->watchdog_timeout_count);
}
static DEVICE_ATTR_RO(watchdog_timeout_count);
static ssize_t clients_show(
struct device *dev, struct device_attribute *attr,
char *buf)
{
struct edgetpu_dev *etdev = dev_get_drvdata(dev);
struct edgetpu_list_device_client *lc;
ssize_t len;
ssize_t ret = 0;
mutex_lock(&etdev->clients_lock);
for_each_list_device_client(etdev, lc) {
struct edgetpu_device_group *group;
mutex_lock(&lc->client->group_lock);
group = lc->client->group;
len = scnprintf(buf, PAGE_SIZE - ret,
"pid %d tgid %d group %d wakelock %d\n",
lc->client->pid, lc->client->tgid,
group ? group->workload_id : -1,
NO_WAKELOCK(lc->client->wakelock) ?
0 : lc->client->wakelock->req_count);
mutex_unlock(&lc->client->group_lock);
buf += len;
ret += len;
}
mutex_unlock(&etdev->clients_lock);
return ret;
}
static DEVICE_ATTR_RO(clients);
static ssize_t show_group(struct edgetpu_dev *etdev,
struct edgetpu_device_group *group, char *buf,
ssize_t buflen)
{
enum edgetpu_context_id context =
edgetpu_group_context_id_locked(group);
struct edgetpu_list_group_client *lc;
ssize_t len;
ssize_t ret = 0;
len = scnprintf(buf, buflen - ret, "group %u ", group->workload_id);
buf += len;
ret += len;
switch (group->status) {
case EDGETPU_DEVICE_GROUP_WAITING:
len = scnprintf(buf, buflen - ret, "forming ");
buf += len;
ret += len;
break;
case EDGETPU_DEVICE_GROUP_FINALIZED:
break;
case EDGETPU_DEVICE_GROUP_ERRORED:
len = scnprintf(buf, buflen - ret, "error %#x ",
group->fatal_errors);
buf += len;
ret += len;
break;
case EDGETPU_DEVICE_GROUP_DISBANDED:
len = scnprintf(buf, buflen - ret, "disbanded\n");
ret += len;
return ret;
}
if (context == EDGETPU_CONTEXT_INVALID)
len = scnprintf(buf, buflen - ret, "context (none) ");
else if (context & EDGETPU_CONTEXT_DOMAIN_TOKEN)
len = scnprintf(buf, buflen - ret, "context detached %#x ",
context & ~(EDGETPU_CONTEXT_DOMAIN_TOKEN));
else
len = scnprintf(buf, buflen - ret, "context mbox %d ",
context);
buf += len;
ret += len;
len = scnprintf(buf, buflen - ret, "vcid %u %s%s\n",
group->vcid, group->dev_inaccessible ? "i" : "",
group->ext_mailbox ? "x" : "");
buf += len;
ret += len;
for_each_list_group_client(lc, group) {
len = scnprintf(buf, buflen - ret, "client %s %d:%d\n",
lc->client->etiface->name,
lc->client->pid, lc->client->tgid);
buf += len;
ret += len;
}
len = scnprintf(buf, buflen - ret, "mappings %zd %zdB\n",
group->host_mappings.count +
group->dmabuf_mappings.count,
edgetpu_group_mappings_total_size(group));
buf += len;
ret += len;
return ret;
}
static ssize_t groups_show(
struct device *dev, struct device_attribute *attr,
char *buf)
{
struct edgetpu_dev *etdev = dev_get_drvdata(dev);
struct edgetpu_device_group *group;
struct edgetpu_list_group *lg;
ssize_t ret = 0;
mutex_lock(&etdev->groups_lock);
etdev_for_each_group(etdev, lg, group) {
edgetpu_device_group_get(group);
ret += show_group(etdev, group, buf + ret, PAGE_SIZE - ret);
edgetpu_device_group_put(group);
}
mutex_unlock(&etdev->groups_lock);
return ret;
}
static DEVICE_ATTR_RO(groups);
static struct attribute *edgetpu_dev_attrs[] = {
&dev_attr_firmware_crash_count.attr,
&dev_attr_watchdog_timeout_count.attr,
&dev_attr_clients.attr,
&dev_attr_groups.attr,
NULL,
};
static const struct attribute_group edgetpu_attr_group = {
.attrs = edgetpu_dev_attrs,
};
const struct file_operations edgetpu_fops = {
.owner = THIS_MODULE,
.llseek = no_llseek,
.mmap = edgetpu_fs_mmap,
.open = edgetpu_fs_open,
.release = edgetpu_fs_release,
.unlocked_ioctl = edgetpu_fs_ioctl,
};
static int edgeptu_fs_add_interface(struct edgetpu_dev *etdev, struct edgetpu_dev_iface *etiface,
const struct edgetpu_iface_params *etiparams)
{
char dev_name[EDGETPU_DEVICE_NAME_MAX];
int ret;
etiface->name = etiparams->name ? etiparams->name : etdev->dev_name;
snprintf(dev_name, EDGETPU_DEVICE_NAME_MAX, "%s",
etiface->name);
dev_dbg(etdev->dev, "adding interface: %s", dev_name);
etiface->devno = MKDEV(MAJOR(edgetpu_basedev),
atomic_add_return(1, &char_minor));
cdev_init(&etiface->cdev, &edgetpu_fops);
ret = cdev_add(&etiface->cdev, etiface->devno, 1);
if (ret) {
dev_err(etdev->dev, "%s: error %d adding cdev for dev %d:%d\n",
etdev->dev_name, ret, MAJOR(etiface->devno),
MINOR(etiface->devno));
return ret;
}
etiface->etcdev = device_create(edgetpu_class, etdev->dev, etiface->devno,
etdev, "%s", dev_name);
if (IS_ERR(etiface->etcdev)) {
ret = PTR_ERR(etiface->etcdev);
dev_err(etdev->dev, "%s: failed to create char device: %d\n",
dev_name, ret);
cdev_del(&etiface->cdev);
return ret;
}
if (etiparams->name)
etiface->d_entry =
debugfs_create_symlink(etiparams->name, edgetpu_debugfs_dir,
etdev->dev_name);
return 0;
}
/* Called from edgetpu core to add new edgetpu device files. */
int edgetpu_fs_add(struct edgetpu_dev *etdev, const struct edgetpu_iface_params *etiparams,
int num_ifaces)
{
int ret;
int i;
etdev->num_ifaces = 0;
dev_dbg(etdev->dev, "%s: adding %u interfaces\n", __func__, num_ifaces);
for (i = 0; i < num_ifaces; i++) {
etdev->etiface[i].etdev = etdev;
ret = edgeptu_fs_add_interface(etdev, &etdev->etiface[i], &etiparams[i]);
if (ret)
return ret;
etdev->num_ifaces++;
}
ret = device_add_group(etdev->dev, &edgetpu_attr_group);
edgetpu_fs_setup_debugfs(etdev);
if (ret)
etdev_warn(etdev, "edgetpu attr group create failed: %d", ret);
return 0;
}
void edgetpu_fs_remove(struct edgetpu_dev *etdev)
{
int i;
device_remove_group(etdev->dev, &edgetpu_attr_group);
for (i = 0; i < etdev->num_ifaces; i++) {
struct edgetpu_dev_iface *etiface = &etdev->etiface[i];
debugfs_remove(etiface->d_entry);
device_destroy(edgetpu_class, etiface->devno);
etiface->etcdev = NULL;
cdev_del(&etiface->cdev);
}
debugfs_remove_recursive(etdev->d_entry);
}
static void edgetpu_debugfs_global_setup(void)
{
edgetpu_debugfs_dir = debugfs_create_dir("edgetpu", NULL);
if (IS_ERR_OR_NULL(edgetpu_debugfs_dir)) {
pr_warn(DRIVER_NAME " error creating edgetpu debugfs dir\n");
return;
}
}
int __init edgetpu_fs_init(void)
{
int ret;
edgetpu_class = class_create(THIS_MODULE, "edgetpu");
if (IS_ERR(edgetpu_class)) {
pr_err(DRIVER_NAME " error creating edgetpu class: %ld\n",
PTR_ERR(edgetpu_class));
return PTR_ERR(edgetpu_class);
}
ret = alloc_chrdev_region(&edgetpu_basedev, 0, EDGETPU_DEV_MAX,
DRIVER_NAME);
if (ret) {
pr_err(DRIVER_NAME " char driver registration failed: %d\n",
ret);
class_destroy(edgetpu_class);
return ret;
}
pr_debug(DRIVER_NAME " registered major=%d\n", MAJOR(edgetpu_basedev));
edgetpu_debugfs_global_setup();
return 0;
}
void __exit edgetpu_fs_exit(void)
{
debugfs_remove_recursive(edgetpu_debugfs_dir);
unregister_chrdev_region(edgetpu_basedev, EDGETPU_DEV_MAX);
class_destroy(edgetpu_class);
}
struct dentry *edgetpu_fs_debugfs_dir(void)
{
return edgetpu_debugfs_dir;
}
MODULE_DESCRIPTION("Google EdgeTPU file operations");
MODULE_VERSION(DRIVER_VERSION);
MODULE_LICENSE("GPL v2");
#ifdef GIT_REPO_TAG
MODULE_INFO(gitinfo, GIT_REPO_TAG);
#endif