blob: f62fe8ad0b857aee851f2c34f52854fee5632070 [file] [log] [blame]
/* Copyright (c) 2008-2016, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
* only version 2 as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
*/
#include <linux/module.h>
#include <linux/fb.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/fdtable.h>
#include <linux/list.h>
#include <linux/debugfs.h>
#include <linux/uaccess.h>
#include <linux/interrupt.h>
#include <linux/workqueue.h>
#include <linux/dma-buf.h>
#include <linux/pm_runtime.h>
#include <linux/rbtree.h>
#include <linux/ashmem.h>
#include <linux/major.h>
#include <linux/io.h>
#include <linux/mman.h>
#include <linux/sort.h>
#include <linux/security.h>
#include <linux/compat.h>
#include <asm/cacheflush.h>
#include "kgsl.h"
#include "kgsl_debugfs.h"
#include "kgsl_cffdump.h"
#include "kgsl_log.h"
#include "kgsl_sharedmem.h"
#include "kgsl_device.h"
#include "kgsl_trace.h"
#include "kgsl_sync.h"
#include "adreno.h"
#include "kgsl_compat.h"
#undef MODULE_PARAM_PREFIX
#define MODULE_PARAM_PREFIX "kgsl."
#ifndef arch_mmap_check
#define arch_mmap_check(addr, len, flags) (0)
#endif
#ifndef pgprot_writebackcache
#define pgprot_writebackcache(_prot) (_prot)
#endif
#ifndef pgprot_writethroughcache
#define pgprot_writethroughcache(_prot) (_prot)
#endif
#ifdef CONFIG_ARM_LPAE
#define KGSL_DMA_BIT_MASK DMA_BIT_MASK(64)
#else
#define KGSL_DMA_BIT_MASK DMA_BIT_MASK(32)
#endif
/*
* Define an kmem cache for the memobj structures since we allocate and free
* them so frequently
*/
static struct kmem_cache *memobjs_cache;
static char *ksgl_mmu_type;
module_param_named(mmutype, ksgl_mmu_type, charp, 0);
MODULE_PARM_DESC(ksgl_mmu_type,
"Type of MMU to be used for graphics. Valid values are 'iommu' or 'nommu'");
struct kgsl_dma_buf_meta {
struct dma_buf_attachment *attach;
struct dma_buf *dmabuf;
struct sg_table *table;
};
static void kgsl_mem_entry_detach_process(struct kgsl_mem_entry *entry);
static int kgsl_setup_dma_buf(struct kgsl_mem_entry *entry,
struct kgsl_pagetable *pagetable,
struct kgsl_device *device,
struct dma_buf *dmabuf);
static const struct file_operations kgsl_fops;
static int __kgsl_check_collision(struct kgsl_process_private *private,
struct kgsl_mem_entry *entry,
unsigned long *gpuaddr, unsigned long len,
int flag_top_down);
/*
* The memfree list contains the last N blocks of memory that have been freed.
* On a GPU fault we walk the list to see if the faulting address had been
* recently freed and print out a message to that effect
*/
#define MEMFREE_ENTRIES 512
static DEFINE_SPINLOCK(memfree_lock);
struct memfree_entry {
unsigned long gpuaddr;
unsigned long size;
pid_t pid;
unsigned int flags;
};
static struct {
struct memfree_entry *list;
int head;
int tail;
} memfree;
static int kgsl_memfree_init(void)
{
memfree.list = kzalloc(MEMFREE_ENTRIES * sizeof(struct memfree_entry),
GFP_KERNEL);
return (memfree.list) ? 0 : -ENOMEM;
}
static void kgsl_memfree_exit(void)
{
kfree(memfree.list);
memset(&memfree, 0, sizeof(memfree));
}
int kgsl_memfree_find_entry(pid_t pid, unsigned long *gpuaddr,
unsigned long *size, unsigned int *flags)
{
int ptr;
if (memfree.list == NULL)
return 0;
spin_lock(&memfree_lock);
ptr = memfree.head - 1;
if (ptr < 0)
ptr = MEMFREE_ENTRIES - 1;
/* Walk backwards through the list looking for the last match */
while (ptr != memfree.tail) {
struct memfree_entry *entry = &memfree.list[ptr];
if ((entry->pid == pid) &&
(*gpuaddr >= entry->gpuaddr &&
*gpuaddr < (entry->gpuaddr + entry->size))) {
*gpuaddr = entry->gpuaddr;
*flags = entry->flags;
*size = entry->size;
spin_unlock(&memfree_lock);
return 1;
}
ptr = ptr - 1;
if (ptr < 0)
ptr = MEMFREE_ENTRIES - 1;
}
spin_unlock(&memfree_lock);
return 0;
}
static void kgsl_memfree_add(pid_t pid, unsigned int gpuaddr,
unsigned int size, int flags)
{
struct memfree_entry *entry;
if (memfree.list == NULL)
return;
spin_lock(&memfree_lock);
entry = &memfree.list[memfree.head];
entry->pid = pid;
entry->gpuaddr = gpuaddr;
entry->size = size;
entry->flags = flags;
memfree.head = (memfree.head + 1) % MEMFREE_ENTRIES;
if (memfree.head == memfree.tail)
memfree.tail = (memfree.tail + 1) % MEMFREE_ENTRIES;
spin_unlock(&memfree_lock);
}
int kgsl_readtimestamp(struct kgsl_device *device, void *priv,
enum kgsl_timestamp_type type, unsigned int *timestamp)
{
return device->ftbl->readtimestamp(device, priv, type, timestamp);
}
EXPORT_SYMBOL(kgsl_readtimestamp);
static inline struct kgsl_mem_entry *
kgsl_mem_entry_create(void)
{
struct kgsl_mem_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
if (entry)
kref_init(&entry->refcount);
return entry;
}
#ifdef CONFIG_DMA_SHARED_BUFFER
static void kgsl_destroy_ion(struct kgsl_dma_buf_meta *meta)
{
if (meta != NULL) {
dma_buf_unmap_attachment(meta->attach, meta->table,
DMA_FROM_DEVICE);
dma_buf_detach(meta->dmabuf, meta->attach);
dma_buf_put(meta->dmabuf);
kfree(meta);
}
}
#else
static void kgsl_destroy_ion(struct kgsl_dma_buf_meta *meta)
{
}
#endif
void
kgsl_mem_entry_destroy(struct kref *kref)
{
struct kgsl_mem_entry *entry = container_of(kref,
struct kgsl_mem_entry,
refcount);
unsigned int memtype;
if (entry == NULL)
return;
/* pull out the memtype before the flags get cleared */
memtype = kgsl_memdesc_usermem_type(&entry->memdesc);
/* Detach from process list */
kgsl_mem_entry_detach_process(entry);
if (memtype != KGSL_MEM_ENTRY_KERNEL)
kgsl_driver.stats.mapped -= entry->memdesc.size;
/*
* Ion takes care of freeing the sglist for us so
* clear the sg before freeing the sharedmem so kgsl_sharedmem_free
* doesn't try to free it again
*/
if (memtype == KGSL_MEM_ENTRY_ION)
entry->memdesc.sg = NULL;
if ((memtype == KGSL_MEM_ENTRY_USER || memtype == KGSL_MEM_ENTRY_ASHMEM)
&& !(entry->memdesc.flags & KGSL_MEMFLAGS_GPUREADONLY)) {
int i = 0, j;
struct scatterlist *sg;
struct page *page;
/*
* Mark all of pages in the scatterlist as dirty since they
* were writable by the GPU.
*/
for_each_sg(entry->memdesc.sg, sg, entry->memdesc.sglen, i) {
page = sg_page(sg);
for (j = 0; j < (sg->length >> PAGE_SHIFT); j++)
set_page_dirty(nth_page(page, j));
}
}
kgsl_sharedmem_free(&entry->memdesc);
switch (memtype) {
case KGSL_MEM_ENTRY_PMEM:
case KGSL_MEM_ENTRY_ASHMEM:
if (entry->priv_data)
fput(entry->priv_data);
break;
case KGSL_MEM_ENTRY_ION:
kgsl_destroy_ion(entry->priv_data);
break;
default:
break;
}
kfree(entry);
}
EXPORT_SYMBOL(kgsl_mem_entry_destroy);
/**
* kgsl_mem_entry_track_gpuaddr - Get the entry gpu address space before
* insertion to the process
* @process: the process that owns the memory
* @entry: the memory entry
*
* @returns - 0 on success else error code
*
* This function should be called with processes memory spinlock held
*/
static int
kgsl_mem_entry_track_gpuaddr(struct kgsl_process_private *process,
struct kgsl_mem_entry *entry)
{
int ret = 0;
struct kgsl_pagetable *pagetable = process->pagetable;
assert_spin_locked(&process->mem_lock);
/*
* If cpu=gpu map is used then caller needs to set the
* gpu address
*/
if (kgsl_memdesc_use_cpu_map(&entry->memdesc)) {
if (!entry->memdesc.gpuaddr)
goto done;
} else if (entry->memdesc.gpuaddr) {
WARN_ONCE(1, "gpuaddr assigned w/o holding memory lock\n");
ret = -EINVAL;
goto done;
}
if (kgsl_memdesc_is_secured(&entry->memdesc))
pagetable = pagetable->mmu->securepagetable;
ret = kgsl_mmu_get_gpuaddr(pagetable, &entry->memdesc);
done:
return ret;
}
static void kgsl_mem_entry_commit_mem_list(struct kgsl_process_private *process,
struct kgsl_mem_entry *entry)
{
struct rb_node **node;
struct rb_node *parent = NULL;
if (!entry->memdesc.gpuaddr)
return;
/* Insert mem entry in mem_rb tree */
node = &process->mem_rb.rb_node;
while (*node) {
struct kgsl_mem_entry *cur;
parent = *node;
cur = rb_entry(parent, struct kgsl_mem_entry, node);
if (entry->memdesc.gpuaddr < cur->memdesc.gpuaddr)
node = &parent->rb_left;
else
node = &parent->rb_right;
}
rb_link_node(&entry->node, parent, node);
rb_insert_color(&entry->node, &process->mem_rb);
}
static void kgsl_mem_entry_commit_process(struct kgsl_process_private *process,
struct kgsl_mem_entry *entry)
{
if (!entry)
return;
spin_lock(&entry->priv->mem_lock);
/* Insert mem entry in mem_rb tree */
kgsl_mem_entry_commit_mem_list(process, entry);
/* Replace mem entry in mem_idr using id */
idr_replace(&entry->priv->mem_idr, entry, entry->id);
spin_unlock(&entry->priv->mem_lock);
}
/**
* kgsl_mem_entry_untrack_gpuaddr() - Untrack memory that is previously tracked
* process - Pointer to process private to which memory belongs
* entry - Memory entry to untrack
*
* Function just does the opposite of kgsl_mem_entry_track_gpuaddr. Needs to be
* called with processes spin lock held
*/
static void
kgsl_mem_entry_untrack_gpuaddr(struct kgsl_process_private *process,
struct kgsl_mem_entry *entry)
{
assert_spin_locked(&process->mem_lock);
if (entry->memdesc.gpuaddr) {
kgsl_mmu_put_gpuaddr(entry->memdesc.pagetable,
&entry->memdesc);
rb_erase(&entry->node, &entry->priv->mem_rb);
}
}
/**
* kgsl_mem_entry_attach_process - Attach a mem_entry to its owner process
* @entry: the memory entry
* @process: the owner process
*
* Attach a newly created mem_entry to its owner process so that
* it can be found later. The mem_entry will be added to mem_idr and have
* its 'id' field assigned. If the GPU address has been set, the entry
* will also be added to the mem_rb tree.
*
* @returns - 0 on success or error code on failure.
*/
static int
kgsl_mem_entry_attach_process(struct kgsl_mem_entry *entry,
struct kgsl_device_private *dev_priv)
{
int id;
int ret;
struct kgsl_process_private *process = dev_priv->process_priv;
struct kgsl_pagetable *pagetable;
ret = kgsl_process_private_get(process);
if (!ret)
return -EBADF;
idr_preload(GFP_KERNEL);
spin_lock(&process->mem_lock);
/* Allocate the ID but don't attach the pointer just yet */
id = idr_alloc(&process->mem_idr, NULL, 1, 0, GFP_NOWAIT);
spin_unlock(&process->mem_lock);
idr_preload_end();
if (id < 0) {
ret = id;
goto err_put_proc_priv;
}
entry->id = id;
entry->priv = process;
spin_lock(&process->mem_lock);
ret = kgsl_mem_entry_track_gpuaddr(process, entry);
if (ret)
idr_remove(&process->mem_idr, entry->id);
spin_unlock(&process->mem_lock);
if (ret)
goto err_put_proc_priv;
/* map the memory after unlocking if gpuaddr has been assigned */
if (entry->memdesc.gpuaddr) {
/* if a secured buffer map it to secure global pagetable */
if (kgsl_memdesc_is_secured(&entry->memdesc))
pagetable = process->pagetable->mmu->securepagetable;
else
pagetable = process->pagetable;
entry->memdesc.pagetable = pagetable;
ret = kgsl_mmu_map(pagetable, &entry->memdesc);
if (ret)
kgsl_mem_entry_detach_process(entry);
}
return ret;
err_put_proc_priv:
kgsl_process_private_put(process);
return ret;
}
/* Detach a memory entry from a process and unmap it from the MMU */
static void kgsl_mem_entry_detach_process(struct kgsl_mem_entry *entry)
{
unsigned int type;
if (entry == NULL)
return;
/* Unmap here so that below we can call kgsl_mmu_put_gpuaddr */
kgsl_mmu_unmap(entry->memdesc.pagetable, &entry->memdesc);
spin_lock(&entry->priv->mem_lock);
kgsl_mem_entry_untrack_gpuaddr(entry->priv, entry);
if (entry->id != 0)
idr_remove(&entry->priv->mem_idr, entry->id);
entry->id = 0;
type = kgsl_memdesc_usermem_type(&entry->memdesc);
entry->priv->stats[type].cur -= entry->memdesc.size;
spin_unlock(&entry->priv->mem_lock);
kgsl_process_private_put(entry->priv);
entry->priv = NULL;
}
/**
* kgsl_context_dump() - dump information about a draw context
* @device: KGSL device that owns the context
* @context: KGSL context to dump information about
*
* Dump specific information about the context to the kernel log. Used for
* fence timeout callbacks
*/
void kgsl_context_dump(struct kgsl_context *context)
{
struct kgsl_device *device;
if (_kgsl_context_get(context) == 0)
return;
device = context->device;
if (kgsl_context_detached(context)) {
dev_err(device->dev, " context[%d]: context detached\n",
context->id);
} else if (device->ftbl->drawctxt_dump != NULL)
device->ftbl->drawctxt_dump(device, context);
kgsl_context_put(context);
}
EXPORT_SYMBOL(kgsl_context_dump);
/* Allocate a new context ID */
int _kgsl_get_context_id(struct kgsl_device *device,
struct kgsl_context *context)
{
int id;
idr_preload(GFP_KERNEL);
write_lock(&device->context_lock);
id = idr_alloc(&device->context_idr, context, 1,
KGSL_MEMSTORE_MAX, GFP_NOWAIT);
write_unlock(&device->context_lock);
idr_preload_end();
if (id > 0)
context->id = id;
return id;
}
/**
* kgsl_context_init() - helper to initialize kgsl_context members
* @dev_priv: the owner of the context
* @context: the newly created context struct, should be allocated by
* the device specific drawctxt_create function.
*
* This is a helper function for the device specific drawctxt_create
* function to initialize the common members of its context struct.
* If this function succeeds, reference counting is active in the context
* struct and the caller should kgsl_context_put() it on error.
* If it fails, the caller should just free the context structure
* it passed in.
*/
int kgsl_context_init(struct kgsl_device_private *dev_priv,
struct kgsl_context *context)
{
struct kgsl_device *device = dev_priv->device;
char name[64];
int ret = 0, id;
id = _kgsl_get_context_id(device, context);
if (id == -ENOSPC) {
/*
* Before declaring that there are no contexts left try
* flushing the event workqueue just in case there are
* detached contexts waiting to finish
*/
mutex_unlock(&device->mutex);
flush_workqueue(device->events_wq);
mutex_lock(&device->mutex);
id = _kgsl_get_context_id(device, context);
}
if (id < 0) {
if (id == -ENOSPC)
KGSL_DRV_INFO(device,
"cannot have more than %zu contexts due to memstore limitation\n",
KGSL_MEMSTORE_MAX);
return id;
}
kref_init(&context->refcount);
/*
* Get a refernce to the process private so its not destroyed, until
* the context is destroyed. This will also prevent the pagetable
* from being destroyed
*/
if (!kgsl_process_private_get(dev_priv->process_priv)) {
ret = -EBADF;
goto out;
}
context->device = dev_priv->device;
context->dev_priv = dev_priv;
context->proc_priv = dev_priv->process_priv;
context->tid = task_pid_nr(current);
ret = kgsl_sync_timeline_create(context);
if (ret)
goto out;
snprintf(name, sizeof(name), "context-%d", id);
kgsl_add_event_group(&context->events, context, name,
kgsl_readtimestamp, context);
out:
if (ret) {
write_lock(&device->context_lock);
idr_remove(&dev_priv->device->context_idr, id);
write_unlock(&device->context_lock);
}
return ret;
}
EXPORT_SYMBOL(kgsl_context_init);
/**
* kgsl_context_detach() - Release the "master" context reference
* @context: The context that will be detached
*
* This is called when a context becomes unusable, because userspace
* has requested for it to be destroyed. The context itself may
* exist a bit longer until its reference count goes to zero.
* Other code referencing the context can detect that it has been
* detached by checking the KGSL_CONTEXT_PRIV_DETACHED bit in
* context->priv.
*/
static void kgsl_context_detach(struct kgsl_context *context)
{
struct kgsl_device *device;
if (context == NULL)
return;
/*
* Mark the context as detached to keep others from using
* the context before it gets fully removed, and to make sure
* we don't try to detach twice.
*/
if (test_and_set_bit(KGSL_CONTEXT_PRIV_DETACHED, &context->priv))
return;
device = context->device;
trace_kgsl_context_detach(device, context);
/* we need to hold device mutex to detach */
mutex_lock(&device->mutex);
context->device->ftbl->drawctxt_detach(context);
mutex_unlock(&device->mutex);
/*
* Cancel all pending events after the device-specific context is
* detached, to avoid possibly freeing memory while it is still
* in use by the GPU.
*/
kgsl_cancel_events(device, &context->events);
/* Remove the event group from the list */
kgsl_del_event_group(&context->events);
kgsl_context_put(context);
}
void
kgsl_context_destroy(struct kref *kref)
{
struct kgsl_context *context = container_of(kref, struct kgsl_context,
refcount);
struct kgsl_device *device = context->device;
trace_kgsl_context_destroy(device, context);
BUG_ON(!kgsl_context_detached(context));
write_lock(&device->context_lock);
if (context->id != KGSL_CONTEXT_INVALID) {
/* Clear the timestamps in the memstore during destroy */
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp), 0);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp), 0);
/* clear device power constraint */
if (context->id == device->pwrctrl.constraint.owner_id) {
trace_kgsl_constraint(device,
device->pwrctrl.constraint.type,
device->pwrctrl.active_pwrlevel,
0);
device->pwrctrl.constraint.type = KGSL_CONSTRAINT_NONE;
}
idr_remove(&device->context_idr, context->id);
context->id = KGSL_CONTEXT_INVALID;
}
write_unlock(&device->context_lock);
kgsl_sync_timeline_destroy(context);
kgsl_process_private_put(context->proc_priv);
device->ftbl->drawctxt_destroy(context);
}
struct kgsl_device *kgsl_get_device(int dev_idx)
{
int i;
struct kgsl_device *ret = NULL;
mutex_lock(&kgsl_driver.devlock);
for (i = 0; i < KGSL_DEVICE_MAX; i++) {
if (kgsl_driver.devp[i] && kgsl_driver.devp[i]->id == dev_idx) {
ret = kgsl_driver.devp[i];
break;
}
}
mutex_unlock(&kgsl_driver.devlock);
return ret;
}
EXPORT_SYMBOL(kgsl_get_device);
static struct kgsl_device *kgsl_get_minor(int minor)
{
struct kgsl_device *ret = NULL;
if (minor < 0 || minor >= KGSL_DEVICE_MAX)
return NULL;
mutex_lock(&kgsl_driver.devlock);
ret = kgsl_driver.devp[minor];
mutex_unlock(&kgsl_driver.devlock);
return ret;
}
/**
* kgsl_check_timestamp() - return true if the specified timestamp is retired
* @device: Pointer to the KGSL device to check
* @context: Pointer to the context for the timestamp
* @timestamp: The timestamp to compare
*/
int kgsl_check_timestamp(struct kgsl_device *device,
struct kgsl_context *context, unsigned int timestamp)
{
unsigned int ts_processed;
kgsl_readtimestamp(device, context, KGSL_TIMESTAMP_RETIRED,
&ts_processed);
return (timestamp_cmp(ts_processed, timestamp) >= 0);
}
EXPORT_SYMBOL(kgsl_check_timestamp);
static int kgsl_suspend_device(struct kgsl_device *device, pm_message_t state)
{
int status = -EINVAL;
if (!device)
return -EINVAL;
KGSL_PWR_WARN(device, "suspend start\n");
mutex_lock(&device->mutex);
status = kgsl_pwrctrl_change_state(device, KGSL_STATE_SUSPEND);
mutex_unlock(&device->mutex);
KGSL_PWR_WARN(device, "suspend end\n");
return status;
}
static int kgsl_resume_device(struct kgsl_device *device)
{
if (!device)
return -EINVAL;
KGSL_PWR_WARN(device, "resume start\n");
mutex_lock(&device->mutex);
if (device->state == KGSL_STATE_SUSPEND) {
kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER);
} else if (device->state != KGSL_STATE_INIT) {
/*
* This is an error situation,so wait for the device
* to idle and then put the device to SLUMBER state.
* This will put the device to the right state when
* we resume.
*/
if (device->state == KGSL_STATE_ACTIVE)
device->ftbl->idle(device);
kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER);
KGSL_PWR_ERR(device,
"resume invoked without a suspend\n");
}
mutex_unlock(&device->mutex);
KGSL_PWR_WARN(device, "resume end\n");
return 0;
}
static int kgsl_suspend(struct device *dev)
{
pm_message_t arg = {0};
struct kgsl_device *device = dev_get_drvdata(dev);
return kgsl_suspend_device(device, arg);
}
static int kgsl_resume(struct device *dev)
{
struct kgsl_device *device = dev_get_drvdata(dev);
return kgsl_resume_device(device);
}
static int kgsl_runtime_suspend(struct device *dev)
{
return 0;
}
static int kgsl_runtime_resume(struct device *dev)
{
return 0;
}
const struct dev_pm_ops kgsl_pm_ops = {
.suspend = kgsl_suspend,
.resume = kgsl_resume,
.runtime_suspend = kgsl_runtime_suspend,
.runtime_resume = kgsl_runtime_resume,
};
EXPORT_SYMBOL(kgsl_pm_ops);
int kgsl_suspend_driver(struct platform_device *pdev,
pm_message_t state)
{
struct kgsl_device *device = dev_get_drvdata(&pdev->dev);
return kgsl_suspend_device(device, state);
}
EXPORT_SYMBOL(kgsl_suspend_driver);
int kgsl_resume_driver(struct platform_device *pdev)
{
struct kgsl_device *device = dev_get_drvdata(&pdev->dev);
return kgsl_resume_device(device);
}
EXPORT_SYMBOL(kgsl_resume_driver);
/**
* kgsl_destroy_process_private() - Cleanup function to free process private
* @kref: - Pointer to object being destroyed's kref struct
* Free struct object and all other resources attached to it.
* Since the function can be used when not all resources inside process
* private have been allocated, there is a check to (before each resource
* cleanup) see if the struct member being cleaned is in fact allocated or not.
* If the value is not NULL, resource is freed.
*/
static void kgsl_destroy_process_private(struct kref *kref)
{
struct kgsl_process_private *private = container_of(kref,
struct kgsl_process_private, refcount);
idr_destroy(&private->mem_idr);
idr_destroy(&private->syncsource_idr);
kgsl_mmu_putpagetable(private->pagetable);
kfree(private);
return;
}
void
kgsl_process_private_put(struct kgsl_process_private *private)
{
if (private)
kref_put(&private->refcount, kgsl_destroy_process_private);
}
/**
* kgsl_process_private_find() - Find the process associated with the specified
* name
* @name: pid_t of the process to search for
* Return the process struct for the given ID.
*/
struct kgsl_process_private *kgsl_process_private_find(pid_t pid)
{
struct kgsl_process_private *p, *private = NULL;
mutex_lock(&kgsl_driver.process_mutex);
list_for_each_entry(p, &kgsl_driver.process_list, list) {
if (p->pid == pid) {
if (kgsl_process_private_get(p))
private = p;
break;
}
}
mutex_unlock(&kgsl_driver.process_mutex);
return private;
}
static struct kgsl_process_private *kgsl_process_private_new(
struct kgsl_device *device)
{
struct kgsl_process_private *private;
pid_t tgid = task_tgid_nr(current);
/* Search in the process list */
list_for_each_entry(private, &kgsl_driver.process_list, list) {
if (private->pid == tgid) {
if (!kgsl_process_private_get(private))
private = ERR_PTR(-EINVAL);
return private;
}
}
/* Create a new object */
private = kzalloc(sizeof(struct kgsl_process_private), GFP_KERNEL);
if (private == NULL)
return ERR_PTR(-ENOMEM);
kref_init(&private->refcount);
private->pid = tgid;
get_task_comm(private->comm, current->group_leader);
private->mem_rb = RB_ROOT;
spin_lock_init(&private->mem_lock);
spin_lock_init(&private->syncsource_lock);
idr_init(&private->mem_idr);
idr_init(&private->syncsource_idr);
/* Allocate a pagetable for the new process object */
if (kgsl_mmu_enabled()) {
private->pagetable = kgsl_mmu_getpagetable(&device->mmu, tgid);
if (private->pagetable == NULL) {
idr_destroy(&private->mem_idr);
idr_destroy(&private->syncsource_idr);
kfree(private);
private = ERR_PTR(-ENOMEM);
}
}
return private;
}
static void process_release_memory(struct kgsl_process_private *private)
{
struct kgsl_mem_entry *entry;
int next = 0;
while (1) {
spin_lock(&private->mem_lock);
entry = idr_get_next(&private->mem_idr, &next);
if (entry == NULL) {
spin_unlock(&private->mem_lock);
break;
}
/*
* If the free pending flag is not set it means that user space
* did not free it's reference to this entry, in that case
* free a reference to this entry, other references are from
* within kgsl so they will be freed eventually by kgsl
*/
if (!entry->pending_free) {
entry->pending_free = 1;
spin_unlock(&private->mem_lock);
kgsl_mem_entry_put(entry);
} else {
spin_unlock(&private->mem_lock);
}
next = next + 1;
}
}
static void process_release_sync_sources(struct kgsl_process_private *private)
{
struct kgsl_syncsource *syncsource;
int next = 0;
while (1) {
spin_lock(&private->syncsource_lock);
syncsource = idr_get_next(&private->syncsource_idr, &next);
spin_unlock(&private->syncsource_lock);
if (syncsource == NULL)
break;
kgsl_syncsource_put(syncsource);
next = next + 1;
}
}
static void kgsl_process_private_close(struct kgsl_device_private *dev_priv,
struct kgsl_process_private *private)
{
mutex_lock(&kgsl_driver.process_mutex);
if (--private->fd_count > 0) {
mutex_unlock(&kgsl_driver.process_mutex);
kgsl_process_private_put(private);
return;
}
/*
* If this is the last file on the process take down the debug
* directories and garbage collect any outstanding resources
*/
kgsl_process_uninit_sysfs(private);
debugfs_remove_recursive(private->debug_root);
process_release_sync_sources(private);
kgsl_mmu_detach_pagetable(private->pagetable);
/* Remove the process struct from the master list */
list_del(&private->list);
/*
* Unlock the mutex before releasing the memory - this prevents a
* deadlock with the IOMMU mutex if a page fault occurs
*/
mutex_unlock(&kgsl_driver.process_mutex);
process_release_memory(private);
kgsl_process_private_put(private);
}
static struct kgsl_process_private *kgsl_process_private_open(
struct kgsl_device *device)
{
struct kgsl_process_private *private;
mutex_lock(&kgsl_driver.process_mutex);
private = kgsl_process_private_new(device);
if (IS_ERR(private))
goto done;
/*
* If this is a new process create the debug directories and add it to
* the process list
*/
if (private->fd_count++ == 0) {
int ret = kgsl_process_init_sysfs(device, private);
if (ret) {
kgsl_process_private_put(private);
private = ERR_PTR(ret);
goto done;
}
ret = kgsl_process_init_debugfs(private);
if (ret) {
kgsl_process_uninit_sysfs(private);
kgsl_process_private_put(private);
private = ERR_PTR(ret);
goto done;
}
list_add(&private->list, &kgsl_driver.process_list);
}
done:
mutex_unlock(&kgsl_driver.process_mutex);
return private;
}
static int kgsl_close_device(struct kgsl_device *device)
{
int result = 0;
mutex_lock(&device->mutex);
device->open_count--;
if (device->open_count == 0) {
/* Wait for the active count to go to 0 */
kgsl_active_count_wait(device, 0);
/* Fail if the wait times out */
BUG_ON(atomic_read(&device->active_cnt) > 0);
result = kgsl_pwrctrl_change_state(device, KGSL_STATE_INIT);
}
mutex_unlock(&device->mutex);
return result;
}
static void device_release_contexts(struct kgsl_device_private *dev_priv)
{
struct kgsl_device *device = dev_priv->device;
struct kgsl_context *context;
int next = 0;
while (1) {
read_lock(&device->context_lock);
context = idr_get_next(&device->context_idr, &next);
read_unlock(&device->context_lock);
if (context == NULL)
break;
if (context->dev_priv == dev_priv) {
/*
* Hold a reference to the context in case somebody
* tries to put it while we are detaching
*/
if (_kgsl_context_get(context)) {
kgsl_context_detach(context);
kgsl_context_put(context);
}
}
next = next + 1;
}
}
static int kgsl_release(struct inode *inodep, struct file *filep)
{
struct kgsl_device_private *dev_priv = filep->private_data;
struct kgsl_device *device = dev_priv->device;
int result;
filep->private_data = NULL;
/* Release the contexts for the file */
device_release_contexts(dev_priv);
/* Close down the process wide resources for the file */
kgsl_process_private_close(dev_priv, dev_priv->process_priv);
kfree(dev_priv);
result = kgsl_close_device(device);
pm_runtime_put(&device->pdev->dev);
return result;
}
static int kgsl_open_device(struct kgsl_device *device)
{
int result = 0;
mutex_lock(&device->mutex);
if (device->open_count == 0) {
/*
* active_cnt special case: we are starting up for the first
* time, so use this sequence instead of the kgsl_pwrctrl_wake()
* which will be called by kgsl_active_count_get().
*/
atomic_inc(&device->active_cnt);
kgsl_sharedmem_set(device, &device->memstore, 0, 0,
device->memstore.size);
result = device->ftbl->init(device);
if (result)
goto err;
result = device->ftbl->start(device, 0);
if (result)
goto err;
/*
* Make sure the gates are open, so they don't block until
* we start suspend or FT.
*/
complete_all(&device->hwaccess_gate);
kgsl_pwrctrl_change_state(device, KGSL_STATE_ACTIVE);
kgsl_active_count_put(device);
}
device->open_count++;
err:
if (result) {
kgsl_pwrctrl_change_state(device, KGSL_STATE_INIT);
atomic_dec(&device->active_cnt);
}
mutex_unlock(&device->mutex);
return result;
}
static int kgsl_open(struct inode *inodep, struct file *filep)
{
int result;
struct kgsl_device_private *dev_priv;
struct kgsl_device *device;
unsigned int minor = iminor(inodep);
device = kgsl_get_minor(minor);
BUG_ON(device == NULL);
result = pm_runtime_get_sync(&device->pdev->dev);
if (result < 0) {
KGSL_DRV_ERR(device,
"Runtime PM: Unable to wake up the device, rc = %d\n",
result);
return result;
}
result = 0;
dev_priv = kzalloc(sizeof(struct kgsl_device_private), GFP_KERNEL);
if (dev_priv == NULL) {
result = -ENOMEM;
goto err;
}
dev_priv->device = device;
filep->private_data = dev_priv;
result = kgsl_open_device(device);
if (result)
goto err;
/*
* Get file (per process) private struct. This must be done
* after the first start so that the global pagetable mappings
* are set up before we create the per-process pagetable.
*/
dev_priv->process_priv = kgsl_process_private_open(device);
if (IS_ERR(dev_priv->process_priv)) {
result = PTR_ERR(dev_priv->process_priv);
kgsl_close_device(device);
goto err;
}
err:
if (result) {
filep->private_data = NULL;
kfree(dev_priv);
pm_runtime_put(&device->pdev->dev);
}
return result;
}
/**
* kgsl_sharedmem_find_region() - Find a gpu memory allocation
*
* @private: private data for the process to check.
* @gpuaddr: start address of the region
* @size: size of the region
*
* Find a gpu allocation. Caller must kgsl_mem_entry_put()
* the returned entry when finished using it.
*/
struct kgsl_mem_entry * __must_check
kgsl_sharedmem_find_region(struct kgsl_process_private *private,
unsigned int gpuaddr, size_t size)
{
struct rb_node *node;
if (!private)
return NULL;
if (!kgsl_mmu_gpuaddr_in_range(private->pagetable, gpuaddr))
return NULL;
spin_lock(&private->mem_lock);
node = private->mem_rb.rb_node;
while (node != NULL) {
struct kgsl_mem_entry *entry;
entry = rb_entry(node, struct kgsl_mem_entry, node);
if (kgsl_gpuaddr_in_memdesc(&entry->memdesc, gpuaddr, size)) {
if (!kgsl_mem_entry_get(entry))
break;
spin_unlock(&private->mem_lock);
return entry;
}
if (gpuaddr < entry->memdesc.gpuaddr)
node = node->rb_left;
else if (gpuaddr >=
(entry->memdesc.gpuaddr + entry->memdesc.size))
node = node->rb_right;
else {
spin_unlock(&private->mem_lock);
return NULL;
}
}
spin_unlock(&private->mem_lock);
return NULL;
}
EXPORT_SYMBOL(kgsl_sharedmem_find_region);
/**
* kgsl_sharedmem_find() - Find a gpu memory allocation
*
* @private: private data for the process to check.
* @gpuaddr: start address of the region
*
* Find a gpu allocation. Caller must kgsl_mem_entry_put()
* the returned entry when finished using it.
*/
static inline struct kgsl_mem_entry * __must_check
kgsl_sharedmem_find(struct kgsl_process_private *private, unsigned int gpuaddr)
{
return kgsl_sharedmem_find_region(private, gpuaddr, 1);
}
/**
* kgsl_sharedmem_region_empty() - Check if an addression region is empty
*
* @private: private data for the process to check.
* @gpuaddr: start address of the region
* @size: length of the region.
* @collision_entry: Returns pointer to the colliding memory entry,
* caller's responsibility to take a refcount on this entry
*
* Checks that there are no existing allocations within an address
* region. This function should be called with processes spin lock
* held.
*/
static int
kgsl_sharedmem_region_empty(struct kgsl_process_private *private,
unsigned int gpuaddr, size_t size,
struct kgsl_mem_entry **collision_entry)
{
int result = 1;
unsigned int gpuaddr_end = gpuaddr + size;
struct rb_node *node;
assert_spin_locked(&private->mem_lock);
if (!kgsl_mmu_gpuaddr_in_range(private->pagetable, gpuaddr))
return 0;
/* don't overflow */
if (gpuaddr_end < gpuaddr)
return 0;
node = private->mem_rb.rb_node;
while (node != NULL) {
struct kgsl_mem_entry *entry;
unsigned int memdesc_start, memdesc_end;
entry = rb_entry(node, struct kgsl_mem_entry, node);
memdesc_start = entry->memdesc.gpuaddr;
memdesc_end = memdesc_start
+ kgsl_memdesc_mmapsize(&entry->memdesc);
if (gpuaddr_end <= memdesc_start)
node = node->rb_left;
else if (memdesc_end <= gpuaddr)
node = node->rb_right;
else {
if (collision_entry)
*collision_entry = entry;
result = 0;
break;
}
}
return result;
}
/**
* kgsl_sharedmem_find_id() - find a memory entry by id
* @process: the owning process
* @id: id to find
*
* @returns - the mem_entry or NULL
*
* Caller must kgsl_mem_entry_put() the returned entry, when finished using
* it.
*/
static inline struct kgsl_mem_entry * __must_check
kgsl_sharedmem_find_id(struct kgsl_process_private *process, unsigned int id)
{
int result = 0;
struct kgsl_mem_entry *entry;
spin_lock(&process->mem_lock);
entry = idr_find(&process->mem_idr, id);
if (entry)
result = kgsl_mem_entry_get(entry);
spin_unlock(&process->mem_lock);
if (!result)
return NULL;
return entry;
}
/**
* kgsl_mem_entry_unset_pend() - Unset the pending free flag of an entry
* @entry - The memory entry
*/
static inline void kgsl_mem_entry_unset_pend(struct kgsl_mem_entry *entry)
{
if (entry == NULL)
return;
spin_lock(&entry->priv->mem_lock);
entry->pending_free = 0;
spin_unlock(&entry->priv->mem_lock);
}
/**
* kgsl_mem_entry_set_pend() - Set the pending free flag of a memory entry
* @entry - The memory entry
*
* @returns - true if pending flag was 0 else false
*
* This function will set the pending free flag if it is previously unset. Used
* to prevent race condition between ioctls calling free/freememontimestamp
* on the same entry. Whichever thread set's the flag first will do the free.
*/
static inline bool kgsl_mem_entry_set_pend(struct kgsl_mem_entry *entry)
{
bool ret = false;
if (entry == NULL)
return false;
spin_lock(&entry->priv->mem_lock);
if (!entry->pending_free) {
entry->pending_free = 1;
ret = true;
}
spin_unlock(&entry->priv->mem_lock);
return ret;
}
/*call all ioctl sub functions with driver locked*/
long kgsl_ioctl_device_getproperty(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
int result = 0;
struct kgsl_device_getproperty *param = data;
switch (param->type) {
case KGSL_PROP_VERSION:
{
struct kgsl_version version;
if (param->sizebytes != sizeof(version)) {
result = -EINVAL;
break;
}
version.drv_major = KGSL_VERSION_MAJOR;
version.drv_minor = KGSL_VERSION_MINOR;
version.dev_major = dev_priv->device->ver_major;
version.dev_minor = dev_priv->device->ver_minor;
if (copy_to_user(param->value, &version, sizeof(version)))
result = -EFAULT;
break;
}
case KGSL_PROP_GPU_RESET_STAT:
{
/* Return reset status of given context and clear it */
uint32_t id;
struct kgsl_context *context;
if (param->sizebytes != sizeof(unsigned int)) {
result = -EINVAL;
break;
}
/* We expect the value passed in to contain the context id */
if (copy_from_user(&id, param->value,
sizeof(unsigned int))) {
result = -EFAULT;
break;
}
context = kgsl_context_get_owner(dev_priv, id);
if (!context) {
result = -EINVAL;
break;
}
/*
* Copy the reset status to value which also serves as
* the out parameter
*/
if (copy_to_user(param->value, &(context->reset_status),
sizeof(unsigned int)))
result = -EFAULT;
else {
/* Clear reset status once its been queried */
context->reset_status = KGSL_CTX_STAT_NO_ERROR;
}
kgsl_context_put(context);
break;
}
default:
if (is_compat_task())
result = dev_priv->device->ftbl->getproperty_compat(
dev_priv->device, param->type,
param->value, param->sizebytes);
else
result = dev_priv->device->ftbl->getproperty(
dev_priv->device, param->type,
param->value, param->sizebytes);
}
return result;
}
long kgsl_ioctl_device_setproperty(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
int result = 0;
/* The getproperty struct is reused for setproperty too */
struct kgsl_device_getproperty *param = data;
/* Reroute to compat version if coming from compat_ioctl */
if (is_compat_task())
result = dev_priv->device->ftbl->setproperty_compat(
dev_priv, param->type, param->value,
param->sizebytes);
else if (dev_priv->device->ftbl->setproperty)
result = dev_priv->device->ftbl->setproperty(
dev_priv, param->type, param->value,
param->sizebytes);
return result;
}
long kgsl_ioctl_device_waittimestamp_ctxtid(
struct kgsl_device_private *dev_priv, unsigned int cmd,
void *data)
{
struct kgsl_device_waittimestamp_ctxtid *param = data;
struct kgsl_device *device = dev_priv->device;
long result = -EINVAL;
unsigned int temp_cur_ts = 0;
struct kgsl_context *context;
mutex_lock(&device->mutex);
context = kgsl_context_get_owner(dev_priv, param->context_id);
if (context == NULL)
goto out;
kgsl_readtimestamp(device, context, KGSL_TIMESTAMP_RETIRED,
&temp_cur_ts);
trace_kgsl_waittimestamp_entry(device, context->id, temp_cur_ts,
param->timestamp, param->timeout);
result = device->ftbl->waittimestamp(device, context, param->timestamp,
param->timeout);
kgsl_readtimestamp(device, context, KGSL_TIMESTAMP_RETIRED,
&temp_cur_ts);
trace_kgsl_waittimestamp_exit(device, temp_cur_ts, result);
out:
kgsl_context_put(context);
mutex_unlock(&device->mutex);
return result;
}
/*
* KGSL command batch management
* A command batch is a single submission from userland. The cmdbatch
* encapsulates everything about the submission : command buffers, flags and
* sync points.
*
* Sync points are events that need to expire before the
* cmdbatch can be queued to the hardware. For each sync point a
* kgsl_cmdbatch_sync_event struct is created and added to a list in the
* cmdbatch. There can be multiple types of events both internal ones (GPU
* events) and external triggers. As the events expire the struct is deleted
* from the list. The GPU will submit the command batch as soon as the list
* goes empty indicating that all the sync points have been met.
*/
void kgsl_dump_syncpoints(struct kgsl_device *device,
struct kgsl_cmdbatch *cmdbatch)
{
struct kgsl_cmdbatch_sync_event *event;
/* Print all the pending sync objects */
list_for_each_entry(event, &cmdbatch->synclist, node) {
switch (event->type) {
case KGSL_CMD_SYNCPOINT_TYPE_TIMESTAMP: {
unsigned int retired;
kgsl_readtimestamp(event->device,
event->context, KGSL_TIMESTAMP_RETIRED,
&retired);
dev_err(device->dev,
" [timestamp] context %d timestamp %d (retired %d)\n",
event->context->id, event->timestamp,
retired);
break;
}
case KGSL_CMD_SYNCPOINT_TYPE_FENCE:
if (event->handle)
dev_err(device->dev, " fence: [%p] %s\n",
event->handle->fence,
event->handle->name);
else
dev_err(device->dev, " fence: invalid\n");
break;
}
}
}
static void _kgsl_cmdbatch_timer(unsigned long data)
{
struct kgsl_device *device;
struct kgsl_cmdbatch *cmdbatch = (struct kgsl_cmdbatch *) data;
struct kgsl_cmdbatch_sync_event *event;
if (cmdbatch == NULL || cmdbatch->context == NULL)
return;
device = cmdbatch->context->device;
dev_err(device->dev,
"kgsl: possible gpu syncpoint deadlock for context %d timestamp %d\n",
cmdbatch->context->id, cmdbatch->timestamp);
set_bit(CMDBATCH_FLAG_FENCE_LOG, &cmdbatch->priv);
kgsl_context_dump(cmdbatch->context);
clear_bit(CMDBATCH_FLAG_FENCE_LOG, &cmdbatch->priv);
spin_lock(&cmdbatch->lock);
/* Print all the fences */
list_for_each_entry(event, &cmdbatch->synclist, node) {
if (KGSL_CMD_SYNCPOINT_TYPE_FENCE == event->type &&
event->handle && event->handle->fence)
kgsl_sync_fence_log(event->handle->fence);
}
spin_unlock(&cmdbatch->lock);
dev_err(device->dev, "--gpu syncpoint deadlock print end--\n");
}
/**
* kgsl_cmdbatch_sync_event_destroy() - Destroy a sync event object
* @kref: Pointer to the kref structure for this object
*
* Actually destroy a sync event object. Called from
* kgsl_cmdbatch_sync_event_put.
*/
static void kgsl_cmdbatch_sync_event_destroy(struct kref *kref)
{
struct kgsl_cmdbatch_sync_event *event = container_of(kref,
struct kgsl_cmdbatch_sync_event, refcount);
kgsl_cmdbatch_put(event->cmdbatch);
kfree(event);
}
/**
* kgsl_cmdbatch_sync_event_put() - Decrement the refcount for a
* sync event object
* @event: Pointer to the sync event object
*/
static inline void kgsl_cmdbatch_sync_event_put(
struct kgsl_cmdbatch_sync_event *event)
{
kref_put(&event->refcount, kgsl_cmdbatch_sync_event_destroy);
}
/**
* kgsl_cmdbatch_destroy_object() - Destroy a cmdbatch object
* @kref: Pointer to the kref structure for this object
*
* Actually destroy a command batch object. Called from kgsl_cmdbatch_put
*/
void kgsl_cmdbatch_destroy_object(struct kref *kref)
{
struct kgsl_cmdbatch *cmdbatch = container_of(kref,
struct kgsl_cmdbatch, refcount);
kgsl_context_put(cmdbatch->context);
kfree(cmdbatch);
}
EXPORT_SYMBOL(kgsl_cmdbatch_destroy_object);
/*
* a generic function to retire a pending sync event and (possibly)
* kick the dispatcher
*/
static void kgsl_cmdbatch_sync_expire(struct kgsl_device *device,
struct kgsl_cmdbatch_sync_event *event)
{
struct kgsl_cmdbatch_sync_event *e, *tmp;
int sched = 0;
int removed = 0;
/*
* We may have cmdbatch timer running, which also uses same lock,
* take a lock with software interrupt disabled (bh) to avoid
* spin lock recursion.
*/
spin_lock_bh(&event->cmdbatch->lock);
/*
* sync events that are contained by a cmdbatch which has been
* destroyed may have already been removed from the synclist
*/
list_for_each_entry_safe(e, tmp, &event->cmdbatch->synclist, node) {
if (e == event) {
list_del_init(&event->node);
removed = 1;
break;
}
}
sched = list_empty(&event->cmdbatch->synclist) ? 1 : 0;
spin_unlock_bh(&event->cmdbatch->lock);
/* If the list is empty delete the canary timer */
if (sched)
del_timer_sync(&event->cmdbatch->timer);
/*
* if this is the last event in the list then tell
* the GPU device that the cmdbatch can be submitted
*/
if (sched && device->ftbl->drawctxt_sched)
device->ftbl->drawctxt_sched(device, event->cmdbatch->context);
/* Put events that have been removed from the synclist */
if (removed)
kgsl_cmdbatch_sync_event_put(event);
}
/*
* This function is called by the GPU event when the sync event timestamp
* expires
*/
static void kgsl_cmdbatch_sync_func(struct kgsl_device *device,
struct kgsl_event_group *group, void *priv, int result)
{
struct kgsl_cmdbatch_sync_event *event = priv;
trace_syncpoint_timestamp_expire(event->cmdbatch,
event->context, event->timestamp);
kgsl_cmdbatch_sync_expire(device, event);
kgsl_context_put(event->context);
/* Put events that have signaled */
kgsl_cmdbatch_sync_event_put(event);
}
static inline void _free_memobj_list(struct list_head *list)
{
struct kgsl_memobj_node *mem, *tmpmem;
/* Free the cmd mem here */
list_for_each_entry_safe(mem, tmpmem, list, node) {
list_del_init(&mem->node);
kmem_cache_free(memobjs_cache, mem);
}
}
/**
* kgsl_cmdbatch_destroy() - Destroy a cmdbatch structure
* @cmdbatch: Pointer to the command batch object to destroy
*
* Start the process of destroying a command batch. Cancel any pending events
* and decrement the refcount. Asynchronous events can still signal after
* kgsl_cmdbatch_destroy has returned.
*/
void kgsl_cmdbatch_destroy(struct kgsl_cmdbatch *cmdbatch)
{
struct kgsl_cmdbatch_sync_event *event, *tmpsync;
LIST_HEAD(cancel_synclist);
int sched = 0;
/* Zap the canary timer */
del_timer_sync(&cmdbatch->timer);
/* non-bh because we just destroyed timer */
spin_lock(&cmdbatch->lock);
/* Empty the synclist before canceling events */
list_splice_init(&cmdbatch->synclist, &cancel_synclist);
spin_unlock(&cmdbatch->lock);
/*
* Finish canceling events outside the cmdbatch spinlock and
* require the cancel function to return if the event was
* successfully canceled meaning that the event is guaranteed
* not to signal the callback. This guarantee ensures that
* the reference count for the event and cmdbatch is correct.
*/
list_for_each_entry_safe(event, tmpsync, &cancel_synclist, node) {
sched = 1;
if (event->type == KGSL_CMD_SYNCPOINT_TYPE_TIMESTAMP) {
/*
* Timestamp events are guaranteed to signal
* when canceled
*/
kgsl_cancel_event(cmdbatch->device,
&event->context->events, event->timestamp,
kgsl_cmdbatch_sync_func, event);
} else if (event->type == KGSL_CMD_SYNCPOINT_TYPE_FENCE) {
/* Put events that are successfully canceled */
if (kgsl_sync_fence_async_cancel(event->handle))
kgsl_cmdbatch_sync_event_put(event);
}
/* Put events that have been removed from the synclist */
list_del_init(&event->node);
kgsl_cmdbatch_sync_event_put(event);
}
/*
* Release the the refcount on the mem entry associated with the
* cmdbatch profiling buffer
*/
if (cmdbatch->flags & KGSL_CMDBATCH_PROFILING)
kgsl_mem_entry_put(cmdbatch->profiling_buf_entry);
/* Destroy the cmdlist we created */
_free_memobj_list(&cmdbatch->cmdlist);
/* Destroy the memlist we created */
_free_memobj_list(&cmdbatch->memlist);
/*
* If we cancelled an event, there's a good chance that the context is
* on a dispatcher queue, so schedule to get it removed.
*/
if (sched && cmdbatch->device->ftbl->drawctxt_sched)
cmdbatch->device->ftbl->drawctxt_sched(cmdbatch->device,
cmdbatch->context);
kgsl_cmdbatch_put(cmdbatch);
}
EXPORT_SYMBOL(kgsl_cmdbatch_destroy);
/*
* A callback that gets registered with kgsl_sync_fence_async_wait and is fired
* when a fence is expired
*/
static void kgsl_cmdbatch_sync_fence_func(void *priv)
{
struct kgsl_cmdbatch_sync_event *event = priv;
trace_syncpoint_fence_expire(event->cmdbatch,
event->handle ? event->handle->name : "unknown");
kgsl_cmdbatch_sync_expire(event->device, event);
/* Put events that have signaled */
kgsl_cmdbatch_sync_event_put(event);
}
/* kgsl_cmdbatch_add_sync_fence() - Add a new sync fence syncpoint
* @device: KGSL device
* @cmdbatch: KGSL cmdbatch to add the sync point to
* @priv: Private sructure passed by the user
*
* Add a new fence sync syncpoint to the cmdbatch.
*/
static int kgsl_cmdbatch_add_sync_fence(struct kgsl_device *device,
struct kgsl_cmdbatch *cmdbatch, void *priv)
{
struct kgsl_cmd_syncpoint_fence *sync = priv;
struct kgsl_cmdbatch_sync_event *event;
event = kzalloc(sizeof(*event), GFP_KERNEL);
if (event == NULL)
return -ENOMEM;
kref_get(&cmdbatch->refcount);
event->type = KGSL_CMD_SYNCPOINT_TYPE_FENCE;
event->cmdbatch = cmdbatch;
event->device = device;
event->context = NULL;
/*
* Initial kref is to ensure async callback does not free the
* event before this function sets the event handle
*/
kref_init(&event->refcount);
/*
* Add it to the list first to account for the possiblity that the
* callback will happen immediately after the call to
* kgsl_sync_fence_async_wait. Decrement the event refcount when
* removing from the synclist.
*/
kref_get(&event->refcount);
/* non-bh because, we haven't started cmdbatch timer yet */
spin_lock(&cmdbatch->lock);
list_add(&event->node, &cmdbatch->synclist);
spin_unlock(&cmdbatch->lock);
/*
* Increment the reference count for the async callback.
* Decrement when the callback is successfully canceled, when
* the callback is signaled or if the async wait fails.
*/
kref_get(&event->refcount);
event->handle = kgsl_sync_fence_async_wait(sync->fd,
kgsl_cmdbatch_sync_fence_func, event);
if (IS_ERR_OR_NULL(event->handle)) {
int ret = PTR_ERR(event->handle);
event->handle = NULL;
/* Failed to add the event to the async callback */
kgsl_cmdbatch_sync_event_put(event);
/* Remove event from the synclist */
spin_lock(&cmdbatch->lock);
list_del(&event->node);
spin_unlock(&cmdbatch->lock);
kgsl_cmdbatch_sync_event_put(event);
/* Event no longer needed by this function */
kgsl_cmdbatch_sync_event_put(event);
/*
* If ret == 0 the fence was already signaled - print a trace
* message so we can track that
*/
if (ret == 0)
trace_syncpoint_fence_expire(cmdbatch, "signaled");
return ret;
}
trace_syncpoint_fence(cmdbatch, event->handle->name);
/*
* Event was successfully added to the synclist, the async
* callback and handle to cancel event has been set.
*/
kgsl_cmdbatch_sync_event_put(event);
return 0;
}
/* kgsl_cmdbatch_add_sync_timestamp() - Add a new sync point for a cmdbatch
* @device: KGSL device
* @cmdbatch: KGSL cmdbatch to add the sync point to
* @priv: Private sructure passed by the user
*
* Add a new sync point timestamp event to the cmdbatch.
*/
static int kgsl_cmdbatch_add_sync_timestamp(struct kgsl_device *device,
struct kgsl_cmdbatch *cmdbatch, void *priv)
{
struct kgsl_cmd_syncpoint_timestamp *sync = priv;
struct kgsl_context *context = kgsl_context_get(cmdbatch->device,
sync->context_id);
struct kgsl_cmdbatch_sync_event *event;
int ret = -EINVAL;
if (context == NULL)
return -EINVAL;
/*
* We allow somebody to create a sync point on their own context.
* This has the effect of delaying a command from submitting until the
* dependent command has cleared. That said we obviously can't let them
* create a sync point on a future timestamp.
*/
if (context == cmdbatch->context) {
unsigned int queued;
kgsl_readtimestamp(device, context, KGSL_TIMESTAMP_QUEUED,
&queued);
if (timestamp_cmp(sync->timestamp, queued) > 0) {
KGSL_DRV_ERR(device,
"Cannot create syncpoint for future timestamp %d (current %d)\n",
sync->timestamp, queued);
goto done;
}
}
event = kzalloc(sizeof(*event), GFP_KERNEL);
if (event == NULL) {
ret = -ENOMEM;
goto done;
}
kref_get(&cmdbatch->refcount);
event->type = KGSL_CMD_SYNCPOINT_TYPE_TIMESTAMP;
event->cmdbatch = cmdbatch;
event->context = context;
event->timestamp = sync->timestamp;
event->device = device;
/*
* Two krefs are required to support events. The first kref is for
* the synclist which holds the event in the cmdbatch. The second
* kref is for the callback which can be asynchronous and be called
* after kgsl_cmdbatch_destroy. The kref should be put when the event
* is removed from the synclist, if the callback is successfully
* canceled or when the callback is signaled.
*/
kref_init(&event->refcount);
kref_get(&event->refcount);
/* non-bh because, we haven't started cmdbatch timer yet */
spin_lock(&cmdbatch->lock);
list_add(&event->node, &cmdbatch->synclist);
spin_unlock(&cmdbatch->lock);
ret = kgsl_add_event(device, &context->events, sync->timestamp,
kgsl_cmdbatch_sync_func, event);
if (ret) {
spin_lock(&cmdbatch->lock);
list_del(&event->node);
spin_unlock(&cmdbatch->lock);
kgsl_cmdbatch_put(cmdbatch);
kfree(event);
} else {
trace_syncpoint_timestamp(cmdbatch, context, sync->timestamp);
}
done:
if (ret)
kgsl_context_put(context);
return ret;
}
/**
* kgsl_cmdbatch_add_sync() - Add a sync point to a command batch
* @device: Pointer to the KGSL device struct for the GPU
* @cmdbatch: Pointer to the cmdbatch
* @sync: Pointer to the user-specified struct defining the syncpoint
*
* Create a new sync point in the cmdbatch based on the user specified
* parameters
*/
int kgsl_cmdbatch_add_sync(struct kgsl_device *device,
struct kgsl_cmdbatch *cmdbatch,
struct kgsl_cmd_syncpoint *sync)
{
void *priv;
int ret, psize;
int (*func)(struct kgsl_device *device, struct kgsl_cmdbatch *cmdbatch,
void *priv);
switch (sync->type) {
case KGSL_CMD_SYNCPOINT_TYPE_TIMESTAMP:
psize = sizeof(struct kgsl_cmd_syncpoint_timestamp);
func = kgsl_cmdbatch_add_sync_timestamp;
break;
case KGSL_CMD_SYNCPOINT_TYPE_FENCE:
psize = sizeof(struct kgsl_cmd_syncpoint_fence);
func = kgsl_cmdbatch_add_sync_fence;
break;
default:
KGSL_DRV_ERR(device, "Invalid sync type 0x%x\n", sync->type);
return -EINVAL;
}
if (sync->size != psize) {
KGSL_DRV_ERR(device, "Invalid sync size %zd\n", sync->size);
return -EINVAL;
}
priv = kzalloc(sync->size, GFP_KERNEL);
if (priv == NULL)
return -ENOMEM;
if (copy_from_user(priv, sync->priv, sync->size)) {
kfree(priv);
return -EFAULT;
}
ret = func(device, cmdbatch, priv);
kfree(priv);
return ret;
}
/**
* kgsl_cmdbatch_add_memobj() - Add an entry to a command batch
* @cmdbatch: Pointer to the cmdbatch
* @ibdesc: Pointer to the user-specified struct defining the memory or IB
* @preamble: Flag to mark this ibdesc as a preamble (if known)
*
* Create a new memory entry in the cmdbatch based on the user specified
* parameters
*/
int kgsl_cmdbatch_add_memobj(struct kgsl_cmdbatch *cmdbatch,
struct kgsl_ibdesc *ibdesc)
{
struct kgsl_memobj_node *mem;
mem = kmem_cache_alloc(memobjs_cache, GFP_KERNEL);
if (mem == NULL)
return -ENOMEM;
mem->gpuaddr = ibdesc->gpuaddr;
mem->sizedwords = ibdesc->sizedwords;
mem->priv = 0;
/* sanitize the ibdesc ctrl flags */
ibdesc->ctrl &= KGSL_IBDESC_MEMLIST | KGSL_IBDESC_PROFILING_BUFFER;
if (cmdbatch->flags & KGSL_CMDBATCH_MEMLIST &&
ibdesc->ctrl & KGSL_IBDESC_MEMLIST) {
/* add to the memlist */
list_add_tail(&mem->node, &cmdbatch->memlist);
/*
* If the memlist contains a cmdbatch profiling buffer, store
* the mem_entry containing the buffer and the gpuaddr at
* which the buffer can be found
*/
if (cmdbatch->flags & KGSL_CMDBATCH_PROFILING &&
ibdesc->ctrl & KGSL_IBDESC_PROFILING_BUFFER &&
!cmdbatch->profiling_buf_entry) {
cmdbatch->profiling_buf_entry =
kgsl_sharedmem_find_region(
cmdbatch->context->proc_priv, mem->gpuaddr,
mem->sizedwords << 2);
if (!cmdbatch->profiling_buf_entry) {
WARN_ONCE(1,
"No mem entry for profiling buf, gpuaddr=%lx\n",
mem->gpuaddr);
return 0;
}
cmdbatch->profiling_buffer_gpuaddr = mem->gpuaddr;
}
} else {
/* set the preamble flag if directed to */
if (cmdbatch->context->flags & KGSL_CONTEXT_PREAMBLE &&
list_empty(&cmdbatch->cmdlist))
mem->priv = MEMOBJ_PREAMBLE;
/* add to the cmd list */
list_add_tail(&mem->node, &cmdbatch->cmdlist);
}
return 0;
}
/**
* kgsl_cmdbatch_create() - Create a new cmdbatch structure
* @device: Pointer to a KGSL device struct
* @context: Pointer to a KGSL context struct
* @flags: Flags for the cmdbatch
*
* Allocate an new cmdbatch structure
*/
static struct kgsl_cmdbatch *kgsl_cmdbatch_create(struct kgsl_device *device,
struct kgsl_context *context, unsigned int flags)
{
struct kgsl_cmdbatch *cmdbatch = kzalloc(sizeof(*cmdbatch), GFP_KERNEL);
if (cmdbatch == NULL)
return ERR_PTR(-ENOMEM);
/*
* Increase the reference count on the context so it doesn't disappear
* during the lifetime of this command batch
*/
if (!_kgsl_context_get(context)) {
kfree(cmdbatch);
return ERR_PTR(-EINVAL);
}
kref_init(&cmdbatch->refcount);
INIT_LIST_HEAD(&cmdbatch->cmdlist);
INIT_LIST_HEAD(&cmdbatch->synclist);
INIT_LIST_HEAD(&cmdbatch->memlist);
spin_lock_init(&cmdbatch->lock);
cmdbatch->device = device;
cmdbatch->context = context;
/* sanitize our flags for cmdbatches */
cmdbatch->flags = flags & (KGSL_CMDBATCH_CTX_SWITCH
| KGSL_CMDBATCH_MARKER
| KGSL_CMDBATCH_END_OF_FRAME
| KGSL_CMDBATCH_SYNC
| KGSL_CMDBATCH_PWR_CONSTRAINT
| KGSL_CMDBATCH_MEMLIST
| KGSL_CMDBATCH_PROFILING);
/* Add a timer to help debug sync deadlocks */
setup_timer(&cmdbatch->timer, _kgsl_cmdbatch_timer,
(unsigned long) cmdbatch);
return cmdbatch;
}
/**
* _kgsl_cmdbatch_verify() - Perform a quick sanity check on a command batch
* @device: Pointer to a KGSL instance that owns the command batch
* @pagetable: Pointer to the pagetable for the current process
* @cmdbatch: Number of indirect buffers to make room for in the cmdbatch
*
* Do a quick sanity test on the list of indirect buffers in a command batch
* verifying that the size and GPU address
*/
static bool _kgsl_cmdbatch_verify(struct kgsl_device_private *dev_priv,
struct kgsl_cmdbatch *cmdbatch)
{
struct kgsl_process_private *private = dev_priv->process_priv;
struct kgsl_memobj_node *ib;
list_for_each_entry(ib, &cmdbatch->cmdlist, node) {
if (ib->sizedwords == 0) {
KGSL_DRV_ERR(dev_priv->device,
"invalid size ctx %d %lX/%zX\n",
cmdbatch->context->id,
ib->gpuaddr,
ib->sizedwords);
return false;
}
if (!kgsl_mmu_gpuaddr_in_range(private->pagetable,
ib->gpuaddr)) {
KGSL_DRV_ERR(dev_priv->device,
"Invalid address ctx %d %lX/%zX\n",
cmdbatch->context->id,
ib->gpuaddr,
ib->sizedwords);
return false;
}
}
return true;
}
/**
* _kgsl_cmdbatch_create_legacy() - Create a cmdbatch from a legacy ioctl struct
* @device: Pointer to the KGSL device struct for the GPU
* @context: Pointer to the KGSL context that issued the command batch
* @param: Pointer to the kgsl_ringbuffer_issueibcmds struct that the user sent
*
* Create a command batch from the legacy issueibcmds format.
*/
static struct kgsl_cmdbatch *_kgsl_cmdbatch_create_legacy(
struct kgsl_device *device,
struct kgsl_context *context,
struct kgsl_ringbuffer_issueibcmds *param)
{
struct kgsl_memobj_node *mem;
struct kgsl_cmdbatch *cmdbatch =
kgsl_cmdbatch_create(device, context, param->flags);
if (IS_ERR(cmdbatch))
return cmdbatch;
mem = kmem_cache_alloc(memobjs_cache, GFP_KERNEL);
if (mem == NULL) {
kgsl_cmdbatch_destroy(cmdbatch);
return ERR_PTR(-ENOMEM);
}
mem->gpuaddr = param->ibdesc_addr;
mem->sizedwords = param->numibs;
mem->priv = 0;
list_add_tail(&mem->node, &cmdbatch->cmdlist);
return cmdbatch;
}
/**
* _kgsl_cmdbatch_create() - Create a cmdbatch from a ioctl struct
* @device: Pointer to the KGSL device struct for the GPU
* @context: Pointer to the KGSL context that issued the command batch
* @flags: Flags passed in from the user command
* @cmdlist: Pointer to the list of commands from the user
* @numcmds: Number of commands in the list
* @synclist: Pointer to the list of syncpoints from the user
* @numsyncs: Number of syncpoints in the list
*
* Create a command batch from the standard issueibcmds format sent by the user.
*/
static struct kgsl_cmdbatch *_kgsl_cmdbatch_create(struct kgsl_device *device,
struct kgsl_context *context, unsigned int flags,
void __user *cmdlist, unsigned int numcmds,
void __user *synclist, unsigned int numsyncs)
{
struct kgsl_cmdbatch *cmdbatch =
kgsl_cmdbatch_create(device, context, flags);
int ret = 0, i;
if (IS_ERR(cmdbatch))
return cmdbatch;
if (is_compat_task()) {
ret = kgsl_cmdbatch_create_compat(device, flags, cmdbatch,
cmdlist, numcmds, synclist, numsyncs);
goto done;
}
if (!(flags & (KGSL_CMDBATCH_SYNC | KGSL_CMDBATCH_MARKER))) {
struct kgsl_ibdesc ibdesc;
void __user *uptr = cmdlist;
for (i = 0; i < numcmds; i++) {
memset(&ibdesc, 0, sizeof(ibdesc));
if (copy_from_user(&ibdesc, uptr, sizeof(ibdesc))) {
ret = -EFAULT;
goto done;
}
ret = kgsl_cmdbatch_add_memobj(cmdbatch, &ibdesc);
if (ret)
goto done;
uptr += sizeof(ibdesc);
}
if (cmdbatch->profiling_buf_entry == NULL)
cmdbatch->flags &= ~KGSL_CMDBATCH_PROFILING;
}
if (synclist && numsyncs) {
struct kgsl_cmd_syncpoint sync;
void __user *uptr = synclist;
for (i = 0; i < numsyncs; i++) {
memset(&sync, 0, sizeof(sync));
if (copy_from_user(&sync, uptr, sizeof(sync))) {
ret = -EFAULT;
goto done;
}
ret = kgsl_cmdbatch_add_sync(device, cmdbatch, &sync);
if (ret)
goto done;
uptr += sizeof(sync);
}
}
done:
if (ret) {
kgsl_cmdbatch_destroy(cmdbatch);
return ERR_PTR(ret);
}
return cmdbatch;
}
long kgsl_ioctl_rb_issueibcmds(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_ringbuffer_issueibcmds *param = data;
struct kgsl_device *device = dev_priv->device;
struct kgsl_context *context;
struct kgsl_cmdbatch *cmdbatch;
long result = -EINVAL;
/* The legacy functions don't support synchronization commands */
if ((param->flags & (KGSL_CMDBATCH_SYNC | KGSL_CMDBATCH_MARKER)))
return -EINVAL;
/* Get the context */
context = kgsl_context_get_owner(dev_priv, param->drawctxt_id);
if (context == NULL)
goto done;
if (param->flags & KGSL_CMDBATCH_SUBMIT_IB_LIST) {
/*
* Do a quick sanity check on the number of IBs in the
* submission
*/
if (param->numibs == 0 || param->numibs > KGSL_MAX_NUMIBS)
goto done;
cmdbatch = _kgsl_cmdbatch_create(device, context, param->flags,
(void __user *)param->ibdesc_addr,
param->numibs, 0, 0);
} else
cmdbatch = _kgsl_cmdbatch_create_legacy(device, context, param);
if (IS_ERR(cmdbatch)) {
result = PTR_ERR(cmdbatch);
goto done;
}
/* Run basic sanity checking on the command */
if (!_kgsl_cmdbatch_verify(dev_priv, cmdbatch))
goto free_cmdbatch;
result = dev_priv->device->ftbl->issueibcmds(dev_priv, context,
cmdbatch, &param->timestamp);
free_cmdbatch:
/*
* -EPROTO is a "success" error - it just tells the user that the
* context had previously faulted
*/
if (result && result != -EPROTO)
kgsl_cmdbatch_destroy(cmdbatch);
done:
kgsl_context_put(context);
return result;
}
long kgsl_ioctl_submit_commands(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_submit_commands *param = data;
struct kgsl_device *device = dev_priv->device;
struct kgsl_context *context;
struct kgsl_cmdbatch *cmdbatch;
long result = -EINVAL;
/*
* The SYNC bit is supposed to identify a dummy sync object so warn the
* user if they specified any IBs with it. A MARKER command can either
* have IBs or not but if the command has 0 IBs it is automatically
* assumed to be a marker. If none of the above make sure that the user
* specified a sane number of IBs
*/
if ((param->flags & KGSL_CMDBATCH_SYNC) && param->numcmds)
KGSL_DEV_ERR_ONCE(device,
"Commands specified with the SYNC flag. They will be ignored\n");
else if (param->numcmds > KGSL_MAX_NUMIBS)
return -EINVAL;
else if (!(param->flags & KGSL_CMDBATCH_SYNC) && param->numcmds == 0)
param->flags |= KGSL_CMDBATCH_MARKER;
context = kgsl_context_get_owner(dev_priv, param->context_id);
if (context == NULL)
return -EINVAL;
cmdbatch = _kgsl_cmdbatch_create(device, context, param->flags,
param->cmdlist, param->numcmds,
param->synclist, param->numsyncs);
if (IS_ERR(cmdbatch)) {
result = PTR_ERR(cmdbatch);
goto done;
}
/* Run basic sanity checking on the command */
if (!_kgsl_cmdbatch_verify(dev_priv, cmdbatch))
goto free_cmdbatch;
result = dev_priv->device->ftbl->issueibcmds(dev_priv, context,
cmdbatch, &param->timestamp);
free_cmdbatch:
/*
* -EPROTO is a "success" error - it just tells the user that the
* context had previously faulted
*/
if (result && result != -EPROTO)
kgsl_cmdbatch_destroy(cmdbatch);
done:
kgsl_context_put(context);
return result;
}
long kgsl_ioctl_cmdstream_readtimestamp_ctxtid(struct kgsl_device_private
*dev_priv, unsigned int cmd,
void *data)
{
struct kgsl_cmdstream_readtimestamp_ctxtid *param = data;
struct kgsl_device *device = dev_priv->device;
struct kgsl_context *context;
long result = -EINVAL;
mutex_lock(&device->mutex);
context = kgsl_context_get_owner(dev_priv, param->context_id);
if (context) {
result = kgsl_readtimestamp(device, context,
param->type, &param->timestamp);
trace_kgsl_readtimestamp(device, context->id,
param->type, param->timestamp);
}
kgsl_context_put(context);
mutex_unlock(&device->mutex);
return result;
}
static void kgsl_freemem_event_cb(struct kgsl_device *device,
struct kgsl_event_group *group, void *priv, int result)
{
struct kgsl_context *context = group->context;
struct kgsl_mem_entry *entry = priv;
unsigned int timestamp;
kgsl_readtimestamp(device, context, KGSL_TIMESTAMP_RETIRED, &timestamp);
/* Free the memory for all event types */
trace_kgsl_mem_timestamp_free(device, entry, KGSL_CONTEXT_ID(context),
timestamp, 0);
kgsl_mem_entry_put(entry);
}
long kgsl_ioctl_cmdstream_freememontimestamp_ctxtid(
struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_cmdstream_freememontimestamp_ctxtid *param = data;
struct kgsl_device *device = dev_priv->device;
struct kgsl_context *context;
struct kgsl_mem_entry *entry;
int result = -EINVAL;
unsigned int temp_cur_ts = 0;
/* If the user supplies incorrect type for timestamp, bail. */
if (param->type != KGSL_TIMESTAMP_RETIRED)
return -EINVAL;
context = kgsl_context_get_owner(dev_priv, param->context_id);
if (context == NULL)
goto out;
entry = kgsl_sharedmem_find(dev_priv->process_priv, param->gpuaddr);
if (!entry) {
KGSL_DRV_ERR(device, "invalid gpuaddr 0x%08lX\n",
param->gpuaddr);
goto out;
}
if (!kgsl_mem_entry_set_pend(entry)) {
KGSL_DRV_WARN(device,
"Cannot set pending bit for gpuaddr 0x%08lX\n",
param->gpuaddr);
kgsl_mem_entry_put(entry);
result = -EBUSY;
goto out;
}
kgsl_readtimestamp(device, context, KGSL_TIMESTAMP_RETIRED,
&temp_cur_ts);
trace_kgsl_mem_timestamp_queue(device, entry, context->id, temp_cur_ts,
param->timestamp);
result = kgsl_add_event(dev_priv->device, &context->events,
param->timestamp, kgsl_freemem_event_cb, entry);
if (result)
kgsl_mem_entry_unset_pend(entry);
kgsl_mem_entry_put(entry);
out:
kgsl_context_put(context);
return result;
}
long kgsl_ioctl_drawctxt_create(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
int result = 0;
struct kgsl_drawctxt_create *param = data;
struct kgsl_context *context = NULL;
struct kgsl_device *device = dev_priv->device;
mutex_lock(&device->mutex);
context = device->ftbl->drawctxt_create(dev_priv, &param->flags);
if (IS_ERR(context)) {
result = PTR_ERR(context);
goto done;
}
trace_kgsl_context_create(dev_priv->device, context, param->flags);
param->drawctxt_id = context->id;
done:
mutex_unlock(&device->mutex);
return result;
}
long kgsl_ioctl_drawctxt_destroy(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_drawctxt_destroy *param = data;
struct kgsl_context *context;
context = kgsl_context_get_owner(dev_priv, param->drawctxt_id);
if (context == NULL)
return -EINVAL;
kgsl_context_detach(context);
kgsl_context_put(context);
return 0;
}
static long _sharedmem_free_entry(struct kgsl_mem_entry *entry)
{
if (!kgsl_mem_entry_set_pend(entry)) {
kgsl_mem_entry_put(entry);
return -EBUSY;
}
trace_kgsl_mem_free(entry);
kgsl_memfree_add(entry->priv->pid, entry->memdesc.gpuaddr,
entry->memdesc.size, entry->memdesc.flags);
/*
* First kgsl_mem_entry_put is for the reference that we took in
* this function when calling kgsl_sharedmem_find, second one is
* to free the memory since this is a free ioctl
*/
kgsl_mem_entry_put(entry);
kgsl_mem_entry_put(entry);
return 0;
}
long kgsl_ioctl_sharedmem_free(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_sharedmem_free *param = data;
struct kgsl_process_private *private = dev_priv->process_priv;
struct kgsl_mem_entry *entry = NULL;
entry = kgsl_sharedmem_find(private, param->gpuaddr);
if (!entry) {
KGSL_MEM_INFO(dev_priv->device, "invalid gpuaddr %08lx\n",
param->gpuaddr);
return -EINVAL;
}
return _sharedmem_free_entry(entry);
}
long kgsl_ioctl_gpumem_free_id(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_gpumem_free_id *param = data;
struct kgsl_process_private *private = dev_priv->process_priv;
struct kgsl_mem_entry *entry = NULL;
entry = kgsl_sharedmem_find_id(private, param->id);
if (!entry) {
KGSL_MEM_INFO(dev_priv->device, "invalid id %d\n", param->id);
return -EINVAL;
}
return _sharedmem_free_entry(entry);
}
static inline int _check_region(unsigned long start, unsigned long size,
uint64_t len)
{
uint64_t end = ((uint64_t) start) + size;
return (end > len);
}
#ifdef CONFIG_FB
static int kgsl_get_phys_file(int fd, unsigned long *start, unsigned long *len,
unsigned long *vstart, struct file **filep)
{
struct file *fbfile;
int ret = 0;
dev_t rdev;
struct fb_info *info;
*start = 0;
*vstart = 0;
*len = 0;
*filep = NULL;
fbfile = fget(fd);
if (fbfile == NULL) {
KGSL_CORE_ERR("fget_light failed\n");
return -1;
}
rdev = fbfile->f_dentry->d_inode->i_rdev;
info = MAJOR(rdev) == FB_MAJOR ? registered_fb[MINOR(rdev)] : NULL;
if (info) {
*start = info->fix.smem_start;
*len = info->fix.smem_len;
*vstart = (unsigned long)__va(info->fix.smem_start);
ret = 0;
} else {
KGSL_CORE_ERR("framebuffer minor %d not found\n",
MINOR(rdev));
ret = -1;
}
fput(fbfile);
return ret;
}
static int kgsl_setup_phys_file(struct kgsl_mem_entry *entry,
struct kgsl_pagetable *pagetable,
unsigned int fd, size_t offset, size_t size)
{
int ret;
unsigned long phys, virt, len;
struct file *filep;
ret = kgsl_get_phys_file(fd, &phys, &len, &virt, &filep);
if (ret)
return ret;
ret = -ERANGE;
if (phys == 0)
goto err;
/* Make sure the length of the region, the offset and the desired
* size are all page aligned or bail
*/
if ((len & ~PAGE_MASK) ||
(offset & ~PAGE_MASK) ||
(size & ~PAGE_MASK)) {
KGSL_CORE_ERR("length offset or size is not page aligned\n");
goto err;
}
/* The size or offset can never be greater than the PMEM length */
if (offset >= len || size > len)
goto err;
/* If size is 0, then adjust it to default to the size of the region
* minus the offset. If size isn't zero, then make sure that it will
* fit inside of the region.
*/
if (size == 0)
size = len - offset;
else if (_check_region(offset, size, len))
goto err;
entry->priv_data = filep;
entry->memdesc.pagetable = pagetable;
entry->memdesc.size = size;
entry->memdesc.physaddr = phys + offset;
entry->memdesc.hostptr = (void *) (virt + offset);
/* USE_CPU_MAP is not impemented for PMEM. */
entry->memdesc.flags &= ~KGSL_MEMFLAGS_USE_CPU_MAP;
entry->memdesc.flags |= KGSL_MEMFLAGS_USERMEM_PMEM;
ret = memdesc_sg_phys(&entry->memdesc, phys + offset, size);
if (ret)
goto err;
return 0;
err:
return ret;
}
#else
static int kgsl_setup_phys_file(struct kgsl_mem_entry *entry,
struct kgsl_pagetable *pagetable,
unsigned int fd, unsigned int offset,
size_t size)
{
return -EINVAL;
}
#endif
static int check_vma_flags(struct vm_area_struct *vma,
unsigned int flags)
{
unsigned long flags_requested = (VM_READ | VM_WRITE);
if (flags & KGSL_MEMFLAGS_GPUREADONLY)
flags_requested &= ~VM_WRITE;
if ((vma->vm_flags & flags_requested) == flags_requested)
return 0;
return -EFAULT;
}
static int check_vma(struct vm_area_struct *vma, struct file *vmfile,
struct kgsl_memdesc *memdesc)
{
if (vma == NULL || vma->vm_file != vmfile)
return -EINVAL;
/* userspace may not know the size, in which case use the whole vma */
if (memdesc->size == 0)
memdesc->size = vma->vm_end - vma->vm_start;
/* range checking */
if (vma->vm_start != memdesc->useraddr ||
(memdesc->useraddr + memdesc->size) != vma->vm_end)
return -EINVAL;
return check_vma_flags(vma, memdesc->flags);
}
static int memdesc_sg_virt(struct kgsl_memdesc *memdesc, struct file *vmfile)
{
int ret = 0;
long npages = 0, i;
unsigned long sglen = memdesc->size / PAGE_SIZE;
struct page **pages = NULL;
int write = ((memdesc->flags & KGSL_MEMFLAGS_GPUREADONLY) ? 0 : 1);
if (sglen == 0 || sglen >= LONG_MAX)
return -EINVAL;
pages = kgsl_malloc(sglen * sizeof(struct page *));
if (pages == NULL)
return -ENOMEM;
memdesc->sg = kgsl_malloc(sglen * sizeof(struct scatterlist));
if (memdesc->sg == NULL) {
ret = -ENOMEM;
goto out;
}
memdesc->sglen = sglen;
sg_init_table(memdesc->sg, sglen);
down_read(&current->mm->mmap_sem);
/* If we have vmfile, make sure we map the correct vma and map it all */
if (vmfile != NULL)
ret = check_vma(find_vma(current->mm, memdesc->useraddr),
vmfile, memdesc);
if (ret == 0) {
npages = get_user_pages(current, current->mm, memdesc->useraddr,
sglen, write, 0, pages, NULL);
ret = (npages < 0) ? (int)npages : 0;
}
up_read(&current->mm->mmap_sem);
if (ret)
goto out;
if ((unsigned long) npages != sglen) {
ret = -EINVAL;
goto out;
}
for (i = 0; i < npages; i++)
sg_set_page(&memdesc->sg[i], pages[i], PAGE_SIZE, 0);
out:
if (ret) {
for (i = 0; i < npages; i++)
put_page(pages[i]);
kgsl_free(memdesc->sg);
memdesc->sg = NULL;
}
kgsl_free(pages);
return ret;
}
static int match_file(const void *p, struct file *file, unsigned int fd)
{
/*
* We must return fd + 1 because iterate_fd stops searching on
* non-zero return, but 0 is a valid fd.
*/
return (p == file) ? (fd + 1) : 0;
}
static void _setup_cache_mode(struct kgsl_mem_entry *entry,
struct vm_area_struct *vma)
{
unsigned int mode;
pgprot_t pgprot = vma->vm_page_prot;
if (pgprot == pgprot_noncached(pgprot))
mode = KGSL_CACHEMODE_UNCACHED;
else if (pgprot == pgprot_writecombine(pgprot))
mode = KGSL_CACHEMODE_WRITECOMBINE;
else
mode = KGSL_CACHEMODE_WRITEBACK;
entry->memdesc.flags |= (mode << KGSL_CACHEMODE_SHIFT);
}
static int kgsl_setup_useraddr(struct kgsl_mem_entry *entry,
struct kgsl_pagetable *pagetable,
void *data,
struct kgsl_device *device)