Google Git
Sign in
android / kernel / google-modules / edgetpu / janeiro / 77305054ef96576b24605b3c303570cd2e34b07b / . / drivers / edgetpu / edgetpu-device-group.c
blob: c410ebc1de04fbe1f5a850d09f150d21b3b24d55 [file] [log] [blame]
// SPDX-License-Identifier: GPL-2.0
/*
* Implements utilities for virtual device group of EdgeTPU.
*
* Copyright (C) 2019 Google, Inc.
*/
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/cred.h>
#include <linux/dma-direction.h>
#include <linux/dma-mapping.h>
#include <linux/eventfd.h>
#include <linux/iommu.h>
#include <linux/kconfig.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/mm.h>
#include <linux/refcount.h>
#include <linux/scatterlist.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/uaccess.h>
#include <linux/uidgid.h>
#include "edgetpu-async.h"
#include "edgetpu-config.h"
#include "edgetpu-device-group.h"
#include "edgetpu-dmabuf.h"
#include "edgetpu-dram.h"
#include "edgetpu-internal.h"
#include "edgetpu-iremap-pool.h"
#include "edgetpu-kci.h"
#include "edgetpu-mapping.h"
#include "edgetpu-mcp.h"
#include "edgetpu-mmu.h"
#include "edgetpu-sw-watchdog.h"
#include "edgetpu-usr.h"
#include "edgetpu-wakelock.h"
#include "edgetpu.h"
#include "mm-backport.h"
#ifdef EDGETPU_HAS_P2P_MAILBOX
#include "edgetpu-p2p-mailbox.h"
#endif
#define for_each_list_group_client_safe(c, n, group) \
list_for_each_entry_safe(c, n, &group->clients, list)
/* Records the mapping and other fields needed for a host buffer mapping */
struct edgetpu_host_map {
struct edgetpu_mapping map;
/*
* SG tables used for dma_map_sg() on each die in the device group.
*
* It's an array of SG tables if the mapping is a mirrored request,
* otherwise it's a NULL pointer.
* The first slot of @sg_tables is never used since the leader of a
* group uses @map->sgt as its SG table.
*/
struct sg_table *sg_tables;
};
/*
* A helper structure for the return value of find_sg_to_sync().
*/
struct sglist_to_sync {
struct scatterlist *sg;
int nelems;
/*
* The SG that has its length modified by find_sg_to_sync().
* Can be NULL, which means no SG's length was modified.
*/
struct scatterlist *last_sg;
/*
* find_sg_to_sync() will temporarily change the length of @last_sg.
* This is used to restore the length.
*/
unsigned int orig_length;
};
#ifdef EDGETPU_HAS_MCP
/* parameter to be used in async KCI jobs */
struct kci_worker_param {
struct edgetpu_device_group *group;
uint idx;
};
static int edgetpu_kci_join_group_worker(struct kci_worker_param *param)
{
struct edgetpu_device_group *group = param->group;
uint i = param->idx;
struct edgetpu_dev *etdev = edgetpu_device_group_nth_etdev(group, i);
int ret;
etdev_dbg(etdev, "%s: join group %u %u/%u", __func__,
group->workload_id, i + 1, group->n_clients);
ret = edgetpu_kci_join_group(etdev->kci, group->n_clients, i);
if (!ret)
edgetpu_sw_wdt_inc_active_ref(etdev);
return ret;
}
static int edgetpu_kci_leave_group_worker(struct kci_worker_param *param)
{
struct edgetpu_device_group *group = param->group;
uint i = param->idx;
struct edgetpu_dev *etdev = edgetpu_device_group_nth_etdev(group, i);
etdev_dbg(etdev, "%s: leave group %u", __func__, group->workload_id);
edgetpu_sw_wdt_dec_active_ref(etdev);
edgetpu_kci_update_usage(etdev);
edgetpu_kci_leave_group(etdev->kci);
return 0;
}
#endif /* EDGETPU_HAS_MCP */
static int edgetpu_group_activate_external_mailbox(struct edgetpu_device_group *group)
{
if (!group->ext_mailbox)
return 0;
edgetpu_mailbox_reinit_external_mailbox(group);
return edgetpu_mailbox_activate_external_mailbox(group);
}
/*
* Activates the VII mailbox @group owns.
*
* Caller holds group->lock.
*/
static int edgetpu_group_activate(struct edgetpu_device_group *group)
{
u8 mailbox_id;
int ret, i;
struct edgetpu_dev *etdev;
if (edgetpu_group_mailbox_detached_locked(group))
return 0;
mailbox_id = edgetpu_group_context_id_locked(group);
ret = edgetpu_mailbox_activate(group->etdev, mailbox_id, group->mbox_attr.client_priv,
group->vcid, !group->activated);
if (ret) {
etdev_err(group->etdev, "activate mailbox for VCID %d failed with %d", group->vcid,
ret);
} else {
group->activated = true;
for (i = 0; i < group->n_clients; i++) {
etdev = edgetpu_device_group_nth_etdev(group, i);
edgetpu_sw_wdt_inc_active_ref(etdev);
}
}
atomic_inc(&group->etdev->job_count);
return ret;
}
static void edgetpu_group_deactivate_external_mailbox(struct edgetpu_device_group *group)
{
edgetpu_mailbox_deactivate_external_mailbox(group);
edgetpu_mailbox_disable_external_mailbox(group);
}
/*
* Deactivates the VII mailbox @group owns.
*
* Caller holds group->lock.
*/
static void edgetpu_group_deactivate(struct edgetpu_device_group *group)
{
u8 mailbox_id;
int i;
struct edgetpu_dev *etdev;
if (edgetpu_group_mailbox_detached_locked(group))
return;
for (i = 0; i < group->n_clients; i++) {
etdev = edgetpu_device_group_nth_etdev(group, i);
edgetpu_sw_wdt_dec_active_ref(etdev);
}
mailbox_id = edgetpu_group_context_id_locked(group);
edgetpu_mailbox_deactivate(group->etdev, mailbox_id);
}
/*
* Handle KCI chores for device group disband.
*
* For multi-chip architectures: asynchronously send LEAVE_GROUP KCI to all
* devices in @group (and GET_USAGE to update usage stats).
*
* For single-chip, multiple client architectures: send KCI CLOSE_DEVICE
* to the device (and GET_USAGE to update usage stats).
*
* Caller holds group->lock.
*/
static void edgetpu_device_group_kci_leave(struct edgetpu_device_group *group)
{
#ifdef EDGETPU_HAS_MULTI_GROUPS
edgetpu_kci_update_usage_async(group->etdev);
/*
* Theoretically we don't need to check @dev_inaccessible here.
* @dev_inaccessible is true implies the client has wakelock count zero, under such case
* edgetpu_mailbox_deactivate() has been called on releasing the wakelock and therefore this
* edgetpu_group_deactivate() call won't send any KCI.
* Still have a check here in case this function does CSR programming other than calling
* edgetpu_mailbox_deactivate() someday.
*/
if (!group->dev_inaccessible)
edgetpu_group_deactivate(group);
#else /* !EDGETPU_HAS_MULTI_GROUPS */
struct kci_worker_param *params =
kmalloc_array(group->n_clients, sizeof(*params), GFP_KERNEL);
struct edgetpu_async_ctx *ctx = edgetpu_async_alloc_ctx();
int i;
int err;
if (!params || !ctx)
goto out_free;
for (i = 0; i < group->n_clients; i++) {
params[i].group = group;
params[i].idx = i;
err = edgetpu_async_add_job(
ctx, &params[i],
(edgetpu_async_job_t)edgetpu_kci_leave_group_worker);
if (err) {
etdev_err(group->etdev,
"%s: failed to create async job: %d",
__func__, err);
goto out_free;
}
}
err = edgetpu_async_wait(ctx);
if (err)
etdev_err(group->etdev, "%s: failed to execute jobs: %d",
__func__, err);
out_free:
edgetpu_async_free_ctx(ctx);
kfree(params);
#endif /* EDGETPU_HAS_MULTI_GROUPS */
}
/*
* Asynchronously sends JOIN_GROUP KCI command to each device in @group.
*
* Caller holds group->lock.
*/
static int
edgetpu_device_group_kci_finalized(struct edgetpu_device_group *group)
{
#ifdef EDGETPU_HAS_MULTI_GROUPS
return edgetpu_group_activate(group);
#else /* !EDGETPU_HAS_MULTI_GROUPS */
struct kci_worker_param *params =
kmalloc_array(group->n_clients, sizeof(*params), GFP_KERNEL);
struct edgetpu_async_ctx *ctx = edgetpu_async_alloc_ctx();
struct edgetpu_async_ctx *ctx_for_leave = edgetpu_async_alloc_ctx();
uint i;
int ret, val;
struct edgetpu_dev *etdev;
if (!params || !ctx || !ctx_for_leave) {
ret = -ENOMEM;
goto out_free;
}
for (i = 0; i < group->n_clients; i++) {
params[i].group = group;
params[i].idx = i;
etdev = edgetpu_device_group_nth_etdev(group, i);
/*
* fast fail.
* It is safe to access @state field here without holding the
* lock as any unresponsive state will lead us to KCI timeout
* anyway.
*/
if (etdev->state != ETDEV_STATE_GOOD) {
ret = edgetpu_get_state_errno_locked(etdev);
etdev_err(group->etdev, "finalize dev %s state %u (%d)",
etdev->dev_name, etdev->state, ret);
goto out_free;
}
ret = edgetpu_async_add_job(
ctx, &params[i],
(edgetpu_async_job_t)edgetpu_kci_join_group_worker);
if (ret) {
etdev_err(group->etdev,
"finalize fw job for %s returns %d",
etdev->dev_name, ret);
goto out_free;
}
atomic_inc(&etdev->job_count);
}
ret = edgetpu_async_wait(ctx);
if (ret) {
etdev_err(group->etdev, "finalize wait returns %d", ret);
goto out_free;
}
for_each_async_ret(ctx, val, i) {
if (val)
goto out_leave;
}
ret = 0;
goto out_free;
out_leave:
for_each_async_ret(ctx, val, i) {
if (val == 0) {
edgetpu_async_add_job(
ctx_for_leave, &params[i],
(edgetpu_async_job_t)
edgetpu_kci_leave_group_worker);
etdev = edgetpu_device_group_nth_etdev(group, i);
atomic_dec(&etdev->job_count);
} else if (val > 0) {
ret = -EBADMSG;
} else {
ret = val;
}
}
val = edgetpu_async_wait(ctx_for_leave);
/* ENOMEM */
if (val)
ret = val;
out_free:
edgetpu_async_free_ctx(ctx_for_leave);
edgetpu_async_free_ctx(ctx);
kfree(params);
return ret;
#endif /* EDGETPU_HAS_MULTI_GROUPS */
}
/*
* Returns the leader of @group.
*
* Must be called with the lock held.
*/
static struct edgetpu_client *edgetpu_device_group_leader(
struct edgetpu_device_group *group)
{
if (group->n_clients < 1 || edgetpu_device_group_is_disbanded(group))
return NULL;
return list_first_entry(&group->clients,
struct edgetpu_list_group_client, list)->client;
}
static int group_alloc_members(struct edgetpu_device_group *group)
{
struct edgetpu_list_group_client *c;
int i = 0;
group->members = kcalloc(group->n_clients, sizeof(*group->members),
GFP_KERNEL);
if (!group->members)
return -ENOMEM;
for_each_list_group_client(c, group) {
group->members[i] = c->client;
i++;
}
return 0;
}
static void group_release_members(struct edgetpu_device_group *group)
{
kfree(group->members);
group->members = NULL;
}
/*
* Does attach domain, init VII, and set @group->context_id without checking
* @group->mailbox_detachable and whether the mailbox is attached.
*
* Caller holds @group->lock.
*/
static int do_attach_mailbox_locked(struct edgetpu_device_group *group)
{
int ret;
ret = edgetpu_mmu_attach_domain(group->etdev, group->etdomain);
if (ret)
return ret;
ret = edgetpu_mailbox_init_vii(&group->vii, group);
if (ret) {
edgetpu_mmu_detach_domain(group->etdev, group->etdomain);
return ret;
}
group->context_id = group->vii.mailbox->mailbox_id;
return 0;
}
/*
* Does detach domain, remove VII, and invalidate @group->context_id without
* checking @group->mailbox_detachable and whether the mailbox is detached.
*
* Caller holds @group->lock.
*/
static void do_detach_mailbox_locked(struct edgetpu_device_group *group)
{
edgetpu_mailbox_remove_vii(&group->vii);
edgetpu_mmu_detach_domain(group->etdev, group->etdomain);
if (group->etdomain->token != EDGETPU_DOMAIN_TOKEN_END)
group->context_id =
EDGETPU_CONTEXT_DOMAIN_TOKEN | group->etdomain->token;
else
group->context_id = EDGETPU_CONTEXT_INVALID;
}
static inline bool is_finalized_or_errored(struct edgetpu_device_group *group)
{
return edgetpu_device_group_is_finalized(group) ||
edgetpu_device_group_is_errored(group);
}
int edgetpu_group_set_eventfd(struct edgetpu_device_group *group, uint event_id,
int eventfd)
{
struct eventfd_ctx *ctx = eventfd_ctx_fdget(eventfd);
ulong flags;
if (IS_ERR(ctx))
return PTR_ERR(ctx);
if (event_id >= EDGETPU_EVENT_COUNT)
return -EINVAL;
write_lock_irqsave(&group->events.lock, flags);
if (group->events.eventfds[event_id])
eventfd_ctx_put(group->events.eventfds[event_id]);
group->events.eventfds[event_id] = ctx;
write_unlock_irqrestore(&group->events.lock, flags);
return 0;
}
void edgetpu_group_unset_eventfd(struct edgetpu_device_group *group,
uint event_id)
{
ulong flags;
if (event_id >= EDGETPU_EVENT_COUNT)
return;
write_lock_irqsave(&group->events.lock, flags);
if (group->events.eventfds[event_id])
eventfd_ctx_put(group->events.eventfds[event_id]);
group->events.eventfds[event_id] = NULL;
write_unlock_irqrestore(&group->events.lock, flags);
}
static void edgetpu_group_clear_events(struct edgetpu_device_group *group)
{
int event_id;
ulong flags;
write_lock_irqsave(&group->events.lock, flags);
for (event_id = 0; event_id < EDGETPU_EVENT_COUNT; event_id++) {
if (group->events.eventfds[event_id])
eventfd_ctx_put(group->events.eventfds[event_id]);
group->events.eventfds[event_id] = NULL;
}
write_unlock_irqrestore(&group->events.lock, flags);
}
void edgetpu_group_notify(struct edgetpu_device_group *group, uint event_id)
{
if (event_id >= EDGETPU_EVENT_COUNT)
return;
etdev_dbg(group->etdev, "%s: group %u id=%u", __func__,
group->workload_id, event_id);
read_lock(&group->events.lock);
if (group->events.eventfds[event_id])
eventfd_signal(group->events.eventfds[event_id], 1);
read_unlock(&group->events.lock);
}
/*
* Releases all resources the group allocated and mark the group as disbanded.
*
* @group must have a valid leader and members, this method uses its clients to
* release VII and P2P mailboxes, buffer mappings, etc.
*
* The lock of group must be held.
*/
static void edgetpu_device_group_release(struct edgetpu_device_group *group)
{
edgetpu_group_clear_events(group);
if (is_finalized_or_errored(group)) {
edgetpu_device_group_kci_leave(group);
/*
* Mappings clear should be performed after had a handshake with
* the firmware.
*/
edgetpu_mappings_clear_group(group);
edgetpu_usr_release_group(group);
edgetpu_group_remove_remote_dram(group);
#ifdef EDGETPU_HAS_P2P_MAILBOX
edgetpu_p2p_mailbox_release(group);
#endif
edgetpu_mailbox_external_disable_free_locked(group);
edgetpu_mailbox_remove_vii(&group->vii);
group_release_members(group);
}
if (group->etdomain) {
edgetpu_mmu_detach_domain(group->etdev, group->etdomain);
edgetpu_mmu_free_domain(group->etdev, group->etdomain);
}
/* Signal any unsignaled dma fences owned by the group with an error. */
edgetpu_sync_fence_group_shutdown(group);
group->status = EDGETPU_DEVICE_GROUP_DISBANDED;
}
/* Checks if two clients are allowed to be grouped. */
static bool edgetpu_clients_groupable(const struct edgetpu_client *client1,
const struct edgetpu_client *client2)
{
struct edgetpu_dev *etdev1 = client1->etdev, *etdev2 = client2->etdev;
return etdev1->mcp_id == etdev2->mcp_id &&
etdev1->mcp_die_index != etdev2->mcp_die_index;
}
/*
* Checks the die indexes are contiguous. Assumes edgetpu_clients_groupable()
* passed, i.e. all devices belong to the same MCP and have different die
* indexes.
*
* Caller holds @group->lock and ensures edgetpu_device_group_nth_etdev()
* is functional.
*/
static bool edgetpu_group_check_contiguity(struct edgetpu_device_group *group)
{
struct edgetpu_mcp *mcp = edgetpu_mcp_of_etdev(group->etdev);
uint i;
uint fr, to;
uint mcp_n = 0;
if (mcp)
mcp_n = mcp->total_num;
fr = group->etdev->mcp_die_index;
for (i = 1; i < group->n_clients; i++, fr = to) {
struct edgetpu_dev *to_etdev =
edgetpu_device_group_nth_etdev(group, i);
to = to_etdev->mcp_die_index;
if (fr + 1 == to)
continue;
if (!mcp_n || (fr + 1) % mcp_n != to) {
struct edgetpu_dev *fr_etdev =
edgetpu_device_group_nth_etdev(group, i - 1);
etdev_err(group->etdev,
"finalize group not contiguous at %s -> %s",
fr_etdev->dev_name, to_etdev->dev_name);
return false;
}
}
return true;
}
/*
* Inserts @group to the list @etdev->groups.
*
* Returns 0 on success.
* Returns -EAGAIN if group join is currently disabled.
*/
static int edgetpu_dev_add_group(struct edgetpu_dev *etdev,
struct edgetpu_device_group *group)
{
struct edgetpu_list_group *l = kmalloc(sizeof(*l), GFP_KERNEL);
int ret;
if (!l)
return -ENOMEM;
mutex_lock(&etdev->groups_lock);
if (etdev->group_join_lockout) {
ret = -EAGAIN;
goto error_unlock;
}
#ifndef EDGETPU_HAS_MULTI_GROUPS
if (etdev->n_groups >= 1) {
ret = -EBUSY;
goto error_unlock;
}
#endif /* !EDGETPU_HAS_MULTI_GROUPS */
if (group->etdev == etdev) {
u32 vcid_pool = etdev->vcid_pool;
#ifdef EDGETPU_VCID_EXTRA_PARTITION
if (group->mbox_attr.partition_type != EDGETPU_PARTITION_EXTRA)
vcid_pool &= ~BIT(EDGETPU_VCID_EXTRA_PARTITION);
else
vcid_pool &= BIT(EDGETPU_VCID_EXTRA_PARTITION);
#endif
if (!vcid_pool) {
ret = -EBUSY;
goto error_unlock;
}
group->vcid = ffs(vcid_pool) - 1;
etdev->vcid_pool &= ~BIT(group->vcid);
}
l->grp = edgetpu_device_group_get(group);
list_add_tail(&l->list, &etdev->groups);
etdev->n_groups++;
mutex_unlock(&etdev->groups_lock);
return 0;
error_unlock:
mutex_unlock(&etdev->groups_lock);
kfree(l);
return ret;
}
void edgetpu_device_group_put(struct edgetpu_device_group *group)
{
if (!group)
return;
if (refcount_dec_and_test(&group->ref_count))
kfree(group);
}
/* caller must hold @etdev->groups_lock. */
static bool edgetpu_in_any_group_locked(struct edgetpu_dev *etdev)
{
return etdev->n_groups;
}
void edgetpu_device_group_leave(struct edgetpu_client *client)
{
struct edgetpu_device_group *group;
struct edgetpu_list_group *l;
struct edgetpu_list_group_client *cur, *nxt;
bool will_disband = false;
mutex_lock(&client->group_lock);
group = client->group;
if (!group) {
mutex_unlock(&client->group_lock);
return;
}
mutex_lock(&group->lock);
/*
* Disband the group if the leader leaves, or it's finalized and any
* member leaves.
*/
if (edgetpu_device_group_is_waiting(group)) {
if (edgetpu_device_group_leader(group) == client)
will_disband = true;
} else if (is_finalized_or_errored(group)) {
will_disband = true;
}
if (will_disband)
/* release the group before removing any members */
edgetpu_device_group_release(group);
/* removes the client from the group list */
for_each_list_group_client_safe(cur, nxt, group) {
if (cur->client == client) {
list_del(&cur->list);
kfree(cur);
edgetpu_client_put(client);
group->n_clients--;
}
}
edgetpu_device_group_put(client->group);
client->group = NULL;
mutex_unlock(&group->lock);
mutex_unlock(&client->group_lock);
/* remove the group from the client device */
mutex_lock(&client->etdev->groups_lock);
list_for_each_entry(l, &client->etdev->groups, list) {
if (l->grp == group) {
if (group->etdev == client->etdev)
client->etdev->vcid_pool |= BIT(group->vcid);
list_del(&l->list);
edgetpu_device_group_put(l->grp);
kfree(l);
client->etdev->n_groups--;
break;
}
}
mutex_unlock(&client->etdev->groups_lock);
}
struct edgetpu_device_group *
edgetpu_device_group_alloc(struct edgetpu_client *client,
const struct edgetpu_mailbox_attr *attr)
{
static uint cur_workload_id;
int ret;
struct edgetpu_device_group *group;
struct edgetpu_iommu_domain *etdomain;
ret = edgetpu_mailbox_validate_attr(attr);
if (ret)
goto error;
/*
* The client already belongs to a group.
* It's safe not to take client->group_lock as
* edgetpu_device_group_add() will fail if there is race.
*/
if (client->group) {
ret = -EINVAL;
goto error;
}
group = kzalloc(sizeof(*group), GFP_KERNEL);
if (!group) {
ret = -ENOMEM;
goto error;
}
refcount_set(&group->ref_count, 1);
group->workload_id = cur_workload_id++;
INIT_LIST_HEAD(&group->clients);
group->n_clients = 0;
group->status = EDGETPU_DEVICE_GROUP_WAITING;
group->etdev = client->etdev;
group->vii.etdev = client->etdev;
mutex_init(&group->lock);
rwlock_init(&group->events.lock);
INIT_LIST_HEAD(&group->dma_fence_list);
edgetpu_mapping_init(&group->host_mappings);
edgetpu_mapping_init(&group->dmabuf_mappings);
group->mbox_attr = *attr;
if (attr->priority & EDGETPU_PRIORITY_DETACHABLE)
group->mailbox_detachable = true;
etdomain = edgetpu_mmu_alloc_domain(group->etdev);
if (!etdomain) {
ret = -ENOMEM;
goto error_put_group;
}
group->etdomain = etdomain;
if (etdomain->token != EDGETPU_DOMAIN_TOKEN_END)
group->context_id =
EDGETPU_CONTEXT_DOMAIN_TOKEN | etdomain->token;
else
group->context_id = EDGETPU_CONTEXT_INVALID;
/* adds @client as the first entry */
ret = edgetpu_device_group_add(group, client);
if (ret) {
etdev_dbg(group->etdev, "%s: group %u add failed ret=%d",
__func__, group->workload_id, ret);
goto error_free_mmu_domain;
}
return group;
error_free_mmu_domain:
edgetpu_mmu_free_domain(group->etdev, group->etdomain);
error_put_group:
edgetpu_device_group_put(group);
error:
return ERR_PTR(ret);
}
int edgetpu_device_group_add(struct edgetpu_device_group *group,
struct edgetpu_client *client)
{
struct edgetpu_list_group_client *c;
int ret = 0;
mutex_lock(&client->group_lock);
if (client->group) {
mutex_unlock(&client->group_lock);
return -EINVAL;
}
mutex_lock(&group->lock);
if (!edgetpu_device_group_is_waiting(group)) {
ret = -EINVAL;
goto out;
}
for_each_list_group_client(c, group) {
if (!edgetpu_clients_groupable(c->client, client)) {
ret = -EINVAL;
goto out;
}
}
c = kzalloc(sizeof(*c), GFP_KERNEL);
if (!c) {
ret = -ENOMEM;
goto out;
}
ret = edgetpu_dev_add_group(client->etdev, group);
if (ret) {
kfree(c);
goto out;
}
c->client = edgetpu_client_get(client);
list_add_tail(&c->list, &group->clients);
client->idx = group->n_clients;
group->n_clients++;
client->group = edgetpu_device_group_get(group);
etdev_dbg(client->etdev, "%s: added group %u", __func__,
group->workload_id);
out:
mutex_unlock(&group->lock);
mutex_unlock(&client->group_lock);
return ret;
}
bool edgetpu_device_group_is_leader(struct edgetpu_device_group *group,
const struct edgetpu_client *client)
{
bool ret;
mutex_lock(&group->lock);
ret = (edgetpu_device_group_leader(group) == client);
mutex_unlock(&group->lock);
return ret;
}
int edgetpu_device_group_finalize(struct edgetpu_device_group *group)
{
int ret = 0;
bool mailbox_attached = false;
struct edgetpu_client *leader;
mutex_lock(&group->lock);
/* do nothing if the group is finalized */
if (is_finalized_or_errored(group))
goto err_unlock;
if (!edgetpu_device_group_is_waiting(group)) {
etdev_err(group->etdev, "finalize group is not waiting");
ret = -EINVAL;
goto err_unlock;
}
ret = group_alloc_members(group);
if (ret) {
etdev_err(group->etdev,
"finalize alloc members returns %d", ret);
goto err_unlock;
}
/*
* Check the contiguity here but not in edgetpu_clients_groupable()
* because clients may leave before the group being finalized.
*/
if (!edgetpu_group_check_contiguity(group)) {
ret = -EINVAL;
goto err_release_members;
}
leader = group->members[0];
/*
* Initialize VII mailbox if
* 1. mailbox is non-detachable: VII is assigned and has the same life
* cycle as a finalized @group, or
* 2. has non-zero wakelock reference counter: VII should be ready to
* use after group is finalized.
*/
if (!group->mailbox_detachable ||
edgetpu_wakelock_count_locked(leader->wakelock)) {
mailbox_attached = true;
ret = do_attach_mailbox_locked(group);
if (ret) {
etdev_err(group->etdev,
"finalize attach mailbox failed: %d", ret);
goto err_release_members;
}
}
#ifdef EDGETPU_HAS_P2P_MAILBOX
ret = edgetpu_p2p_mailbox_setup(group);
if (ret) {
etdev_err(group->etdev,
"finalize p2p mailbox setup failed: %d", ret);
goto err_detach_mailbox;
}
#endif
ret = edgetpu_group_setup_remote_dram(group);
if (ret) {
etdev_err(group->etdev,
"finalize remote dram setup failed: %d", ret);
goto err_release_p2p;
}
edgetpu_usr_init_group(group);
/* send KCI only if the device is powered on */
if (edgetpu_wakelock_count_locked(leader->wakelock)) {
ret = edgetpu_device_group_kci_finalized(group);
if (ret)
goto err_remove_remote_dram;
}
group->status = EDGETPU_DEVICE_GROUP_FINALIZED;
mutex_unlock(&group->lock);
return 0;
err_remove_remote_dram:
edgetpu_group_remove_remote_dram(group);
err_release_p2p:
#ifdef EDGETPU_HAS_P2P_MAILBOX
edgetpu_p2p_mailbox_release(group);
err_detach_mailbox:
#endif
if (mailbox_attached)
do_detach_mailbox_locked(group);
err_release_members:
group_release_members(group);
err_unlock:
mutex_unlock(&group->lock);
return ret;
}
bool edgetpu_in_any_group(struct edgetpu_dev *etdev)
{
bool ret;
mutex_lock(&etdev->groups_lock);
ret = edgetpu_in_any_group_locked(etdev);
mutex_unlock(&etdev->groups_lock);
return ret;
}
bool edgetpu_set_group_join_lockout(struct edgetpu_dev *etdev, bool lockout)
{
bool ret = true;
mutex_lock(&etdev->groups_lock);
if (lockout && edgetpu_in_any_group_locked(etdev))
ret = false;
else
etdev->group_join_lockout = lockout;
mutex_unlock(&etdev->groups_lock);
return ret;
}
/* parameter to be used in async iova mapping jobs */
struct iova_mapping_worker_param {
struct edgetpu_device_group *group;
struct edgetpu_host_map *hmap;
uint idx;
};
static int edgetpu_map_iova_sgt_worker(void *p)
{
struct iova_mapping_worker_param *param = p;
struct edgetpu_device_group *group = param->group;
uint i = param->idx;
struct edgetpu_host_map *hmap = param->hmap;
const struct edgetpu_mapping *map = &hmap->map;
enum edgetpu_context_id ctx_id = edgetpu_group_context_id_locked(group);
struct edgetpu_dev *etdev = edgetpu_device_group_nth_etdev(group, i);
int ret;
edgetpu_mmu_reserve(etdev, map->alloc_iova, map->alloc_size);
ret = edgetpu_mmu_map_iova_sgt(etdev, map->device_address,
&hmap->sg_tables[i], map->dir,
map_to_mmu_flags(map->flags),
ctx_id);
if (ret)
edgetpu_mmu_free(etdev, map->alloc_iova, map->alloc_size);
return ret;
}
/*
* Requests all devices except the leader in @group to map
* @hmap->map.device_address -> corresponding @hmap->sg_tables[].
*
* The caller holds the group lock.
*
* Returns 0 on success.
* Returns a negative errno on error, no mapping operations are performed in
* this case.
*/
static int edgetpu_device_group_map_iova_sgt(struct edgetpu_device_group *group,
struct edgetpu_host_map *hmap)
{
uint i;
int ret;
int val;
const struct edgetpu_mapping *map = &hmap->map;
enum edgetpu_context_id ctx_id = edgetpu_group_context_id_locked(group);
struct edgetpu_async_ctx *ctx;
struct iova_mapping_worker_param *params;
/* only leader in @group */
if (group->n_clients == 1) {
ret = 0;
goto out;
}
ctx = edgetpu_async_alloc_ctx();
params = kmalloc_array(group->n_clients - 1, sizeof(*params),
GFP_KERNEL);
if (!params || !ctx) {
ret = -ENOMEM;
goto out_free;
}
for (i = 0; i < group->n_clients - 1; i++) {
params[i].hmap = hmap;
params[i].group = group;
params[i].idx = i + 1;
ret = edgetpu_async_add_job(ctx, &params[i],
edgetpu_map_iova_sgt_worker);
if (ret)
goto out_free;
}
ret = edgetpu_async_wait(ctx);
if (ret)
goto out_free;
for_each_async_ret(ctx, val, i) {
if (val) {
ret = val;
goto rollback;
}
}
goto out_free;
rollback:
for_each_async_ret(ctx, val, i) {
if (val == 0) {
struct edgetpu_dev *etdev;
int idx = i + 1;
etdev = edgetpu_device_group_nth_etdev(group, idx);
edgetpu_mmu_unmap_iova_sgt_attrs(
etdev, map->device_address,
&hmap->sg_tables[idx], map->dir, ctx_id,
DMA_ATTR_SKIP_CPU_SYNC);
edgetpu_mmu_free(etdev, map->alloc_iova,
map->alloc_size);
}
}
out_free:
edgetpu_async_free_ctx(ctx);
kfree(params);
out:
return ret;
}
/*
* Removes previously added mapping.
*
* The caller holds the group lock.
*/
static void
edgetpu_device_group_unmap_iova_sgt(struct edgetpu_device_group *group,
struct edgetpu_host_map *hmap)
{
const struct edgetpu_mapping *map = &hmap->map;
struct edgetpu_dev *etdev;
enum edgetpu_context_id ctx_id = edgetpu_group_context_id_locked(group);
uint i;
for (i = 1; i < group->n_clients; i++) {
etdev = edgetpu_device_group_nth_etdev(group, i);
edgetpu_mmu_unmap_iova_sgt_attrs(etdev, map->device_address,
&hmap->sg_tables[i], map->dir,
ctx_id, map->dma_attrs);
edgetpu_mmu_free(etdev, map->alloc_iova, map->alloc_size);
}
}
/*
* Unmap a mapping specified by @map. Unmaps from IOMMU and unpins pages,
* frees mapping node, which is invalid upon return.
*
* Caller locks group->host_mappings.
*/
static void edgetpu_unmap_node(struct edgetpu_mapping *map)
{
struct edgetpu_device_group *group = map->priv;
enum edgetpu_context_id ctx_id = edgetpu_group_context_id_locked(group);
struct edgetpu_host_map *hmap =
container_of(map, struct edgetpu_host_map, map);
struct edgetpu_dev *etdev;
struct sg_page_iter sg_iter;
uint i;
etdev_dbg(group->etdev, "%s: %u: die=%d, iova=%#llx", __func__,
group->workload_id, map->die_index, map->device_address);
if (map->device_address) {
if (IS_MIRRORED(map->flags)) {
etdev = group->etdev;
edgetpu_device_group_unmap_iova_sgt(group, hmap);
} else {
etdev = edgetpu_device_group_nth_etdev(group,
map->die_index);
}
edgetpu_mmu_unmap(etdev, map, ctx_id);
}
for_each_sg_page(map->sgt.sgl, &sg_iter, map->sgt.orig_nents, 0) {
struct page *page = sg_page_iter_page(&sg_iter);
if (map->dir == DMA_FROM_DEVICE ||
map->dir == DMA_BIDIRECTIONAL)
set_page_dirty(page);
unpin_user_page(page);
}
sg_free_table(&map->sgt);
if (IS_MIRRORED(map->flags)) {
for (i = 1; i < group->n_clients; i++)
sg_free_table(&hmap->sg_tables[i]);
kfree(hmap->sg_tables);
}
edgetpu_device_group_put(map->priv);
kfree(hmap);
}
static void edgetpu_host_map_show(struct edgetpu_mapping *map,
struct seq_file *s)
{
struct scatterlist *sg;
int i;
size_t cur_offset = 0;
for_each_sg(map->sgt.sgl, sg, map->sgt.nents, i) {
dma_addr_t phys_addr = sg_phys(sg);
dma_addr_t dma_addr = sg_dma_address(sg);
if (IS_MIRRORED(map->flags))
seq_puts(s, " mirrored: ");
else
seq_printf(s, " die %u: ", map->die_index);
seq_printf(s, "%#llx %lu %s %#llx %pap %pad\n",
map->device_address + cur_offset,
DIV_ROUND_UP(sg_dma_len(sg), PAGE_SIZE),
edgetpu_dma_dir_rw_s(map->dir),
map->host_address + cur_offset, &phys_addr,
&dma_addr);
cur_offset += sg_dma_len(sg);
}
}
size_t edgetpu_group_mappings_total_size(struct edgetpu_device_group *group)
{
return edgetpu_mappings_total_size(&group->host_mappings) +
edgetpu_mappings_total_size(&group->dmabuf_mappings);
}
/*
* Pins the user-space address @arg->host_address and returns the pinned pages.
* @pnum_pages is set to the number of pages.
*
* Returns -errno if failed on pinning @size bytes.
*/
static struct page **edgetpu_pin_user_pages(struct edgetpu_device_group *group,
struct edgetpu_map_ioctl *arg,
uint *pnum_pages, bool *preadonly)
{
u64 host_addr = untagged_addr(arg->host_address);
u64 size = arg->size;
uint num_pages;
ulong offset;
struct edgetpu_dev *etdev = group->etdev;
struct page **pages;
int i;
int ret;
struct vm_area_struct *vma;
struct vm_area_struct **vmas;
unsigned int foll_flags = FOLL_LONGTERM | FOLL_WRITE;
if (size == 0)
return ERR_PTR(-EINVAL);
if (!access_ok((const void *)host_addr, size)) {
etdev_err(etdev, "invalid address range in buffer map request");
return ERR_PTR(-EFAULT);
}
offset = host_addr & (PAGE_SIZE - 1);
num_pages = DIV_ROUND_UP((size + offset), PAGE_SIZE);
if (num_pages * PAGE_SIZE < size + offset)
return ERR_PTR(-EINVAL);
etdev_dbg(etdev, "%s: hostaddr=%#llx pages=%u", __func__, host_addr, num_pages);
/*
* "num_pages" is decided from user-space arguments, don't show warnings
* when facing malicious input.
*/
pages = kvmalloc((num_pages * sizeof(*pages)), GFP_KERNEL | __GFP_NOWARN);
if (!pages) {
etdev_err(etdev, "out of memory allocating pages (%lu bytes)",
num_pages * sizeof(*pages));
return ERR_PTR(-ENOMEM);
}
/*
* The host pages might be read-only and could fail if we attempt to pin
* it with FOLL_WRITE.
* default to read/write if find_extend_vma returns NULL
*/
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0)
down_read(&current->mm->mmap_sem);
#else
mmap_read_lock(current->mm);
#endif
vma = find_extend_vma(current->mm, host_addr & PAGE_MASK);
if (vma && !(vma->vm_flags & VM_WRITE)) {
foll_flags &= ~FOLL_WRITE;
*preadonly = true;
} else {
*preadonly = false;
}
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0)
up_read(&current->mm->mmap_sem);
#else
mmap_read_unlock(current->mm);
#endif
/* Try fast call first, in case it's actually faster. */
ret = pin_user_pages_fast(host_addr & PAGE_MASK, num_pages, foll_flags,
pages);
if (ret == num_pages) {
*pnum_pages = num_pages;
return pages;
}
if (ret == -EFAULT && !*preadonly) {
foll_flags &= ~FOLL_WRITE;
*preadonly = true;
ret = pin_user_pages_fast(host_addr & PAGE_MASK, num_pages,
foll_flags, pages);
}
if (ret < 0) {
etdev_dbg(etdev, "pin_user_pages failed %u:%pK-%u: %d",
group->workload_id, (void *)host_addr, num_pages,
ret);
if (ret == -EFAULT)
etdev_err(etdev,
"bad address locking %u pages for %s",
num_pages, *preadonly ? "read" : "write");
if (ret != -ENOMEM) {
num_pages = 0;
goto error;
}
}
etdev_dbg(etdev,
"pin_user_pages_fast error %u:%pK npages=%u ret=%d",
group->workload_id, (void *)host_addr, num_pages,
ret);
/* Unpin any partial mapping and start over again. */
for (i = 0; i < ret; i++)
unpin_user_page(pages[i]);
/* Allocate our own vmas array non-contiguous. */
vmas = kvmalloc((num_pages * sizeof(*vmas)), GFP_KERNEL | __GFP_NOWARN);
if (!vmas) {
etdev_err(etdev, "out of memory allocating vmas (%lu bytes)",
num_pages * sizeof(*pages));
kvfree(pages);
return ERR_PTR(-ENOMEM);
}
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0)
down_read(&current->mm->mmap_sem);
#else
mmap_read_lock(current->mm);
#endif
ret = pin_user_pages(host_addr & PAGE_MASK, num_pages, foll_flags,
pages, vmas);
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0)
up_read(&current->mm->mmap_sem);
#else
mmap_read_unlock(current->mm);
#endif
kvfree(vmas);
if (ret < 0) {
etdev_dbg(etdev, "pin_user_pages failed %u:%pK-%u: %d",
group->workload_id, (void *)host_addr, num_pages,
ret);
if (ret == -ENOMEM)
etdev_err(etdev,
"system out of memory locking %u pages",
num_pages);
num_pages = 0;
goto error;
}
if (ret < num_pages) {
etdev_dbg(etdev,
"pin_user_pages partial %u:%pK npages=%u pinned=%d",
group->workload_id, (void *)host_addr, num_pages,
ret);
etdev_err(etdev, "can only lock %u of %u pages requested",
(unsigned int)ret, num_pages);
num_pages = ret;
ret = -EFAULT;
goto error;
}
*pnum_pages = num_pages;
return pages;
error:
for (i = 0; i < num_pages; i++)
unpin_user_page(pages[i]);
kvfree(pages);
return ERR_PTR(ret);
}
/*
* Allocates an edgetpu_host_map with the user-space address @host_addr.
*/
static struct edgetpu_host_map *
alloc_mapping_from_useraddr(struct edgetpu_device_group *group, u64 host_addr,
edgetpu_map_flag_t flags, struct page **pages,
uint num_pages)
{
struct edgetpu_dev *etdev = group->etdev;
struct edgetpu_host_map *hmap;
int n;
struct sg_table *sgt;
int i;
int ret;
hmap = kzalloc(sizeof(*hmap), GFP_KERNEL);
if (!hmap) {
ret = -ENOMEM;
goto error;
}
hmap->map.host_address = host_addr;
hmap->map.dir = map_flag_to_host_dma_dir(flags);
hmap->map.priv = edgetpu_device_group_get(group);
hmap->map.release = edgetpu_unmap_node;
hmap->map.show = edgetpu_host_map_show;
hmap->map.flags = flags;
hmap->map.dma_attrs = map_to_dma_attr(flags, true);
if (IS_MIRRORED(flags)) {
hmap->sg_tables = kcalloc(group->n_clients,
sizeof(*hmap->sg_tables), GFP_KERNEL);
if (!hmap->sg_tables) {
ret = -ENOMEM;
goto error;
}
n = group->n_clients;
} else {
n = 1;
}
for (i = 0; i < n; i++) {
if (i == 0)
sgt = &hmap->map.sgt;
else
sgt = &hmap->sg_tables[i];
ret = sg_alloc_table_from_pages(sgt, pages, num_pages, 0,
num_pages * PAGE_SIZE,
GFP_KERNEL);
if (ret) {
etdev_dbg(etdev,
"%s: sg_alloc_table_from_pages failed %u:%pK-%u: %d",
__func__, group->workload_id,
(void *)host_addr, num_pages, ret);
goto error_free_sgt;
}
}
return hmap;
error_free_sgt:
/*
* Starting from kernel version 5.10, the caller must call sg_free_table
* to clean up any leftover allocations if sg_alloc_table_from_pages
* returns non-0 for failures. Calling sg_free_table is also fine with
* older kernel versions since sg_free_table handles this properly.
*/
for (; i >= 0; i--) {
if (i == 0)
sgt = &hmap->map.sgt;
else
sgt = &hmap->sg_tables[i];
sg_free_table(sgt);
}
error:
if (hmap) {
edgetpu_device_group_put(hmap->map.priv);
kfree(hmap->sg_tables);
kfree(hmap);
}
return ERR_PTR(ret);
}
/*
* Finds the scatterlist covering range [start, end).
*
* The found SG and number of elements will be stored in @sglist.
*
* To ensure the returned SG list strictly locates in range [start, end), the
* last SG's length is shrunk. Therefore caller must call
* restore_sg_after_sync(@sglist) after the DMA sync is performed.
*
* @sglist->nelems == 0 means the target range exceeds the whole SG table.
*/
static void find_sg_to_sync(const struct sg_table *sgt, u64 start, u64 end,
struct sglist_to_sync *sglist)
{
struct scatterlist *sg;
size_t cur_offset = 0;
int i;
sglist->sg = NULL;
sglist->nelems = 0;
sglist->last_sg = NULL;
if (unlikely(end == 0))
return;
for_each_sg(sgt->sgl, sg, sgt->orig_nents, i) {
if (cur_offset <= start && start < cur_offset + sg->length)
sglist->sg = sg;
if (sglist->sg)
++sglist->nelems;
cur_offset += sg->length;
if (end <= cur_offset) {
sglist->last_sg = sg;
sglist->orig_length = sg->length;
/*
* To let the returned SG list have exact length as
* [start, end).
*/
sg->length -= cur_offset - end;
break;
}
}
}
static void restore_sg_after_sync(struct sglist_to_sync *sglist)
{
if (!sglist->last_sg)
return;
sglist->last_sg->length = sglist->orig_length;
}
/*
* Performs DMA sync of the mapping with region [offset, offset + size).
*
* Caller holds mapping's lock, to prevent @hmap being modified / removed by
* other processes.
*/
static int group_sync_host_map(struct edgetpu_device_group *group,
struct edgetpu_host_map *hmap, u64 offset,
u64 size, enum dma_data_direction dir,
bool for_cpu)
{
const u64 end = offset + size;
typeof(dma_sync_sg_for_cpu) *sync =
for_cpu ? dma_sync_sg_for_cpu : dma_sync_sg_for_device;
struct edgetpu_dev *etdev;
struct sg_table *sgt;
struct sglist_to_sync sglist;
int i;
sgt = &hmap->map.sgt;
find_sg_to_sync(sgt, offset, end, &sglist);
if (!sglist.nelems)
return -EINVAL;
if (IS_MIRRORED(hmap->map.flags))
etdev = group->etdev;
else
etdev = edgetpu_device_group_nth_etdev(group,
hmap->map.die_index);
sync(etdev->dev, sglist.sg, sglist.nelems, dir);
restore_sg_after_sync(&sglist);
if (IS_MIRRORED(hmap->map.flags)) {
for (i = 1; i < group->n_clients; i++) {
etdev = edgetpu_device_group_nth_etdev(group, i);
find_sg_to_sync(&hmap->sg_tables[i], offset, end,
&sglist);
if (WARN_ON(!sglist.sg))
return -EINVAL;
sync(etdev->dev, sglist.sg, sglist.nelems, dir);
restore_sg_after_sync(&sglist);
}
}
return 0;
}
int edgetpu_device_group_map(struct edgetpu_device_group *group,
struct edgetpu_map_ioctl *arg)
{
uint num_pages = 0;
struct page **pages;
int ret = -EINVAL;
u64 host_addr = arg->host_address;
edgetpu_map_flag_t flags = arg->flags;
struct edgetpu_host_map *hmap;
struct edgetpu_mapping *map = NULL;
struct edgetpu_dev *etdev;
enum edgetpu_context_id context_id;
const u32 mmu_flags = map_to_mmu_flags(flags) | EDGETPU_MMU_HOST;
int i;
bool readonly;
tpu_addr_t tpu_addr;
if (!valid_dma_direction(flags & EDGETPU_MAP_DIR_MASK))
return -EINVAL;
/* Pin user pages before holding any lock. */
pages = edgetpu_pin_user_pages(group, arg, &num_pages, &readonly);
if (IS_ERR(pages))
return PTR_ERR(pages);
/* If the host pages are read-only, fallback to use DMA_TO_DEVICE. */
if (readonly) {
flags &= ~EDGETPU_MAP_DIR_MASK;
flags |= EDGETPU_MAP_DMA_TO_DEVICE;
}
mutex_lock(&group->lock);
context_id = edgetpu_group_context_id_locked(group);
if (!edgetpu_device_group_is_finalized(group)) {
ret = edgetpu_group_errno(group);
goto error;
}
if (!IS_MIRRORED(flags)) {
if (arg->die_index >= group->n_clients) {
ret = -EINVAL;
goto error;
}
}
hmap = alloc_mapping_from_useraddr(group, host_addr, flags, pages,
num_pages);
if (IS_ERR(hmap)) {
ret = PTR_ERR(hmap);
goto error;
}
map = &hmap->map;
if (IS_MIRRORED(flags)) {
map->die_index = ALL_DIES;
etdev = group->etdev;
ret = edgetpu_mmu_map(etdev, map, context_id, mmu_flags);
if (ret)
goto error;
ret = edgetpu_device_group_map_iova_sgt(group, hmap);
if (ret) {
etdev_dbg(etdev,
"group add translation failed %u:%#llx",
group->workload_id, map->device_address);
goto error;
}
} else {
map->die_index = arg->die_index;
etdev = edgetpu_device_group_nth_etdev(group, map->die_index);
ret = edgetpu_mmu_map(etdev, map, context_id, mmu_flags);
if (ret)
goto error;
}
map->map_size = arg->size;
/*
* @map can be freed (by another thread) once it's added to the mappings, record the address
* before that.
*/
tpu_addr = map->device_address;
ret = edgetpu_mapping_add(&group->host_mappings, map);
if (ret) {
etdev_dbg(etdev, "duplicate mapping %u:%#llx", group->workload_id, tpu_addr);
goto error;
}
mutex_unlock(&group->lock);
arg->device_address = tpu_addr;
kvfree(pages);
return 0;
error:
if (map) {
edgetpu_mapping_lock(&group->host_mappings);
/* this will free @hmap */
edgetpu_unmap_node(map);
edgetpu_mapping_unlock(&group->host_mappings);
} else {
/* revert edgetpu_pin_user_pages() */
for (i = 0; i < num_pages; i++)
unpin_user_page(pages[i]);
}
mutex_unlock(&group->lock);
kvfree(pages);
return ret;
}
int edgetpu_device_group_unmap(struct edgetpu_device_group *group,
u32 die_index, tpu_addr_t tpu_addr,
edgetpu_map_flag_t flags)
{
struct edgetpu_mapping *map;
int ret = 0;
mutex_lock(&group->lock);
if (!is_finalized_or_errored(group)) {
ret = -EINVAL;
goto unlock_group;
}
edgetpu_mapping_lock(&group->host_mappings);
map = edgetpu_mapping_find_locked(&group->host_mappings, ALL_DIES,
tpu_addr);
if (!map)
map = edgetpu_mapping_find_locked(&group->host_mappings,
die_index, tpu_addr);
if (!map) {
edgetpu_mapping_unlock(&group->host_mappings);
etdev_dbg(group->etdev,
"%s: mapping not found for workload %u: %#llx",
__func__, group->workload_id, tpu_addr);
ret = -EINVAL;
goto unlock_group;
}
edgetpu_mapping_unlink(&group->host_mappings, map);
map->dma_attrs = map_to_dma_attr(flags, false);
edgetpu_unmap_node(map);
edgetpu_mapping_unlock(&group->host_mappings);
unlock_group:
mutex_unlock(&group->lock);
return ret;
}
int edgetpu_device_group_sync_buffer(struct edgetpu_device_group *group,
const struct edgetpu_sync_ioctl *arg)
{
struct edgetpu_mapping *map;
int ret = 0;
tpu_addr_t tpu_addr = arg->device_address;
/*
* Sync operations don't care the data correctness of prefetch by TPU CPU if they mean to
* sync FROM_DEVICE only, so @dir here doesn't need to be wrapped with host_dma_dir().
*/
enum dma_data_direction dir = arg->flags & EDGETPU_MAP_DIR_MASK;
struct edgetpu_host_map *hmap;
if (!valid_dma_direction(dir))
return -EINVAL;
/* invalid if size == 0 or overflow */
if (arg->offset + arg->size <= arg->offset)
return -EINVAL;
mutex_lock(&group->lock);
if (!edgetpu_device_group_is_finalized(group)) {
ret = edgetpu_group_errno(group);
goto unlock_group;
}
edgetpu_mapping_lock(&group->host_mappings);
map = edgetpu_mapping_find_locked(&group->host_mappings, ALL_DIES,
tpu_addr);
if (!map)
map = edgetpu_mapping_find_locked(&group->host_mappings,
arg->die_index, tpu_addr);
if (!map) {
ret = -EINVAL;
goto unlock_mapping;
}
hmap = container_of(map, struct edgetpu_host_map, map);
ret = group_sync_host_map(group, hmap, arg->offset, arg->size, dir,
arg->flags & EDGETPU_SYNC_FOR_CPU);
unlock_mapping:
edgetpu_mapping_unlock(&group->host_mappings);
unlock_group:
mutex_unlock(&group->lock);
return ret;
}
void edgetpu_mappings_clear_group(struct edgetpu_device_group *group)
{
edgetpu_mapping_clear(&group->host_mappings);
edgetpu_mapping_clear(&group->dmabuf_mappings);
}
void edgetpu_group_mappings_show(struct edgetpu_device_group *group,
struct seq_file *s)
{
enum edgetpu_context_id context =
edgetpu_group_context_id_locked(group);
seq_printf(s, "group %u", group->workload_id);
switch (group->status) {
case EDGETPU_DEVICE_GROUP_WAITING:
case EDGETPU_DEVICE_GROUP_FINALIZED:
break;
case EDGETPU_DEVICE_GROUP_ERRORED:
seq_puts(s, " (errored)");
break;
case EDGETPU_DEVICE_GROUP_DISBANDED:
seq_puts(s, ": disbanded\n");
return;
}
if (context == EDGETPU_CONTEXT_INVALID)
seq_puts(s, " context (none):\n");
else if (context & EDGETPU_CONTEXT_DOMAIN_TOKEN)
seq_printf(s, " context detached %#x:\n",
context & ~(EDGETPU_CONTEXT_DOMAIN_TOKEN));
else
seq_printf(s, " context mbox %d:\n", context);
if (group->host_mappings.count) {
seq_printf(s, "host buffer mappings (%zd):\n",
group->host_mappings.count);
edgetpu_mappings_show(&group->host_mappings, s);
}
if (group->dmabuf_mappings.count) {
seq_printf(s, "dma-buf buffer mappings (%zd):\n",
group->dmabuf_mappings.count);
edgetpu_mappings_show(&group->dmabuf_mappings, s);
}
if (group->vii.cmd_queue_mem.vaddr) {
seq_puts(s, "VII queues:\n");
seq_printf(s, " %#llx %lu cmdq %#llx %pad\n",
group->vii.cmd_queue_mem.tpu_addr,
DIV_ROUND_UP(group->vii.cmd_queue_mem.size,
PAGE_SIZE),
group->vii.cmd_queue_mem.host_addr,
&group->vii.cmd_queue_mem.dma_addr);
seq_printf(s, " %#llx %lu rspq %#llx %pad\n",
group->vii.resp_queue_mem.tpu_addr,
DIV_ROUND_UP(group->vii.resp_queue_mem.size,
PAGE_SIZE),
group->vii.resp_queue_mem.host_addr,
&group->vii.resp_queue_mem.dma_addr);
}
#ifdef EDGETPU_HAS_P2P_MAILBOX
if (group->p2p_mailbox_matrix) {
seq_puts(s, "P2P queues:\n");
edgetpu_p2p_mailbox_show(group, s);
}
#endif
}
int edgetpu_mmap_csr(struct edgetpu_device_group *group,
struct vm_area_struct *vma, bool is_external)
{
struct edgetpu_dev *etdev = group->etdev;
int ret = 0;
ulong phys_base, vma_size, map_size;
if (is_external && !uid_eq(current_euid(), GLOBAL_ROOT_UID))
return -EPERM;
mutex_lock(&group->lock);
if (!edgetpu_group_finalized_and_attached(group)) {
ret = edgetpu_group_errno(group);
goto out;
}
if (is_external && (!group->ext_mailbox || !group->ext_mailbox->descriptors)) {
ret = -ENOENT;
goto out;
}
vma_size = vma->vm_end - vma->vm_start;
map_size = min(vma_size, USERSPACE_CSR_SIZE);
if (is_external)
phys_base = etdev->regs.phys +
group->ext_mailbox->descriptors[0].mailbox->cmd_queue_csr_base;
else
phys_base = etdev->regs.phys + group->vii.mailbox->cmd_queue_csr_base;
ret = io_remap_pfn_range(vma, vma->vm_start, phys_base >> PAGE_SHIFT,
map_size, vma->vm_page_prot);
if (ret)
etdev_dbg(etdev, "Error remapping PFN range: %d", ret);
out:
mutex_unlock(&group->lock);
return ret;
}
int edgetpu_mmap_queue(struct edgetpu_device_group *group,
enum mailbox_queue_type type,
struct vm_area_struct *vma, bool is_external)
{
struct edgetpu_dev *etdev = group->etdev;
int ret = 0;
edgetpu_queue_mem *queue_mem;
if (is_external && !uid_eq(current_euid(), GLOBAL_ROOT_UID))
return -EPERM;
mutex_lock(&group->lock);
if (!edgetpu_group_finalized_and_attached(group)) {
ret = edgetpu_group_errno(group);
goto out;
}
if (is_external && (!group->ext_mailbox || !group->ext_mailbox->descriptors)) {
ret = -ENOENT;
goto out;
}
if (type == MAILBOX_CMD_QUEUE) {
if (is_external)
queue_mem = &(group->ext_mailbox->descriptors[0].cmd_queue_mem);
else
queue_mem = &(group->vii.cmd_queue_mem);
} else {
if (is_external)
queue_mem = &(group->ext_mailbox->descriptors[0].resp_queue_mem);
else
queue_mem = &(group->vii.resp_queue_mem);
}
if (!queue_mem->vaddr) {
ret = -ENXIO;
goto out;
}
ret = edgetpu_iremap_mmap(etdev, vma, queue_mem);
if (!ret)
queue_mem->host_addr = vma->vm_start;
out:
mutex_unlock(&group->lock);
return ret;
}
/*
* Set @group status as errored, set the error mask, and notify the runtime of
* the fatal error event on the group.
*/
void edgetpu_group_fatal_error_notify(struct edgetpu_device_group *group,
uint error_mask)
{
etdev_dbg(group->etdev, "notify group %u error %#x",
group->workload_id, error_mask);
mutex_lock(&group->lock);
/*
* Only finalized groups may have handshake with the FW, mark
* them as errored.
*/
if (edgetpu_device_group_is_finalized(group))
group->status = EDGETPU_DEVICE_GROUP_ERRORED;
group->fatal_errors |= error_mask;
mutex_unlock(&group->lock);
edgetpu_group_notify(group, EDGETPU_EVENT_FATAL_ERROR);
}
/*
* For each group active on @etdev: set the group status as errored, set the
* error mask, and notify the runtime of the fatal error event.
*/
void edgetpu_fatal_error_notify(struct edgetpu_dev *etdev, uint error_mask)
{
size_t i, num_groups = 0;
struct edgetpu_device_group *group;
struct edgetpu_device_group **groups;
struct edgetpu_list_group *g;
mutex_lock(&etdev->groups_lock);
groups = kmalloc_array(etdev->n_groups, sizeof(*groups), GFP_KERNEL);
if (unlikely(!groups)) {
/*
* Just give up setting status in this case, this only happens
* when the system is OOM.
*/
mutex_unlock(&etdev->groups_lock);
return;
}
/*
* Fetch the groups into an array to set the group status without
* holding @etdev->groups_lock. To prevent the potential deadlock that
* edgetpu_device_group_add() holds group->lock then etdev->groups_lock.
*/
etdev_for_each_group(etdev, g, group) {
if (edgetpu_device_group_is_disbanded(group))
continue;
groups[num_groups++] = edgetpu_device_group_get(group);
}
mutex_unlock(&etdev->groups_lock);
for (i = 0; i < num_groups; i++) {
edgetpu_group_fatal_error_notify(groups[i], error_mask);
edgetpu_device_group_put(groups[i]);
}
kfree(groups);
}
uint edgetpu_group_get_fatal_errors(struct edgetpu_device_group *group)
{
uint fatal_errors;
mutex_lock(&group->lock);
fatal_errors = edgetpu_group_get_fatal_errors_locked(group);
mutex_unlock(&group->lock);
return fatal_errors;
}
void edgetpu_group_detach_mailbox_locked(struct edgetpu_device_group *group)
{
if (!group->mailbox_detachable)
return;
if (edgetpu_group_mailbox_detached_locked(group))
return;
do_detach_mailbox_locked(group);
}
void edgetpu_group_close_and_detach_mailbox(struct edgetpu_device_group *group)
{
mutex_lock(&group->lock);
/*
* Only a finalized group may have mailbox attached.
*
* Detaching mailbox for an errored group is also fine.
*/
if (is_finalized_or_errored(group)) {
edgetpu_group_deactivate(group);
edgetpu_group_detach_mailbox_locked(group);
edgetpu_group_deactivate_external_mailbox(group);
}
mutex_unlock(&group->lock);
}
int edgetpu_group_attach_mailbox_locked(struct edgetpu_device_group *group)
{
if (!group->mailbox_detachable)
return 0;
if (!edgetpu_group_mailbox_detached_locked(group))
return 0;
return do_attach_mailbox_locked(group);
}
int edgetpu_group_attach_and_open_mailbox(struct edgetpu_device_group *group)
{
int ret = 0;
mutex_lock(&group->lock);
/*
* Only attaching mailbox for finalized groups.
* Don't attach mailbox for errored groups.
*/
if (!edgetpu_device_group_is_finalized(group))
goto out_unlock;
ret = edgetpu_group_attach_mailbox_locked(group);
if (ret)
goto out_unlock;
ret = edgetpu_group_activate(group);
if (ret)
goto error_detach;
ret = edgetpu_group_activate_external_mailbox(group);
if (!ret)
goto out_unlock;
edgetpu_group_deactivate(group);
error_detach:
edgetpu_group_detach_mailbox_locked(group);
out_unlock:
mutex_unlock(&group->lock);
return ret;
}
/*
* Return the group with id @vcid for device @etdev, with a reference held
* on the group (must call edgetpu_device_group_put when done), or NULL if
* no group with that VCID is found.
*/
static struct edgetpu_device_group *get_group_by_vcid(
struct edgetpu_dev *etdev, u16 vcid)
{
struct edgetpu_device_group *group = NULL;
struct edgetpu_device_group *tgroup;
struct edgetpu_list_group *g;
mutex_lock(&etdev->groups_lock);
etdev_for_each_group(etdev, g, tgroup) {
if (tgroup->vcid == vcid) {
group = edgetpu_device_group_get(tgroup);
break;
}
}
mutex_unlock(&etdev->groups_lock);
return group;
}
void edgetpu_handle_job_lockup(struct edgetpu_dev *etdev, u16 vcid)
{
struct edgetpu_device_group *group;
etdev_err(etdev, "firmware-detected job lockup on VCID %u",
vcid);
group = get_group_by_vcid(etdev, vcid);
if (!group) {
etdev_warn(etdev, "VCID %u group not found", vcid);
return;
}
edgetpu_group_fatal_error_notify(group, EDGETPU_ERROR_RUNTIME_TIMEOUT);
edgetpu_device_group_put(group);
}