| /************************************************************************** |
| * |
| * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, Tx., USA |
| * All Rights Reserved. |
| * Copyright 2009 VMware, Inc., Palo Alto, CA., USA |
| * All Rights Reserved. |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the |
| * "Software"), to deal in the Software without restriction, including |
| * without limitation the rights to use, copy, modify, merge, publish, |
| * distribute, sub license, and/or sell copies of the Software, and to |
| * permit persons to whom the Software is furnished to do so, subject to |
| * the following conditions: |
| * |
| * The above copyright notice and this permission notice (including the |
| * next paragraph) shall be included in all copies or substantial portions |
| * of the Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
| * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
| * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
| * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
| * USE OR OTHER DEALINGS IN THE SOFTWARE. |
| * |
| **************************************************************************/ |
| /* |
| * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com> |
| */ |
| #ifdef HAVE_CONFIG_H |
| #include "config.h" |
| #endif |
| |
| #include <xf86drm.h> |
| #include <stdlib.h> |
| #include <unistd.h> |
| #include <errno.h> |
| #include <sys/mman.h> |
| #include <drm/psb_ttm_placement_user.h> |
| #include "wsbm_pool.h" |
| #include "assert.h" |
| #include "wsbm_priv.h" |
| #include "wsbm_manager.h" |
| |
| #define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \ |
| do { \ |
| (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \ |
| } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ |
| |
| #define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \ |
| do { \ |
| (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \ |
| } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ |
| |
| /* |
| * Buffer pool implementation using DRM buffer objects as wsbm buffer objects. |
| */ |
| |
| struct _TTMBuffer |
| { |
| struct _WsbmBufStorage buf; |
| struct _WsbmCond event; |
| |
| /* |
| * Remains constant after creation. |
| */ |
| |
| uint64_t requestedSize; |
| uint64_t mapHandle; |
| uint64_t realSize; |
| |
| /* |
| * Protected by the kernel lock. |
| */ |
| |
| struct _WsbmKernelBuf kBuf; |
| |
| /* |
| * Protected by the mutex. |
| */ |
| |
| void *virtual; |
| int syncInProgress; |
| unsigned readers; |
| unsigned writers; |
| }; |
| |
| struct _TTMPool |
| { |
| struct _WsbmBufferPool pool; |
| unsigned int pageSize; |
| unsigned int devOffset; |
| }; |
| |
| static inline struct _TTMPool * |
| ttmGetPool(struct _TTMBuffer *dBuf) |
| { |
| return containerOf(dBuf->buf.pool, struct _TTMPool, pool); |
| } |
| |
| static inline struct _TTMBuffer * |
| ttmBuffer(struct _WsbmBufStorage *buf) |
| { |
| return containerOf(buf, struct _TTMBuffer, buf); |
| } |
| |
| static struct _WsbmBufStorage * |
| pool_create(struct _WsbmBufferPool *pool, |
| unsigned long size, uint32_t placement, unsigned alignment) |
| { |
| struct _TTMBuffer *dBuf = (struct _TTMBuffer *) |
| calloc(1, sizeof(*dBuf)); |
| struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); |
| int ret; |
| unsigned pageSize = ttmPool->pageSize; |
| union ttm_pl_create_arg arg; |
| |
| if (!dBuf) |
| return NULL; |
| |
| if ((alignment > pageSize) && (alignment % pageSize)) |
| goto out_err0; |
| |
| ret = wsbmBufStorageInit(&dBuf->buf, pool); |
| if (ret) |
| goto out_err0; |
| |
| ret = WSBM_COND_INIT(&dBuf->event); |
| if (ret) |
| goto out_err1; |
| |
| arg.req.size = size; |
| arg.req.placement = placement; |
| arg.req.page_alignment = alignment / pageSize; |
| |
| DRMRESTARTCOMMANDWRITEREAD(pool->fd, ttmPool->devOffset + TTM_PL_CREATE, |
| arg, ret); |
| |
| if (ret) |
| goto out_err2; |
| |
| dBuf->requestedSize = size; |
| dBuf->kBuf.gpuOffset = arg.rep.gpu_offset; |
| dBuf->mapHandle = arg.rep.map_handle; |
| dBuf->realSize = arg.rep.bo_size; |
| dBuf->kBuf.placement = arg.rep.placement; |
| dBuf->kBuf.handle = arg.rep.handle; |
| |
| return &dBuf->buf; |
| |
| out_err2: |
| WSBM_COND_FREE(&dBuf->event); |
| out_err1: |
| wsbmBufStorageTakedown(&dBuf->buf); |
| out_err0: |
| free(dBuf); |
| return NULL; |
| } |
| |
| static struct _WsbmBufStorage * |
| pool_reference(struct _WsbmBufferPool *pool, unsigned handle) |
| { |
| struct _TTMBuffer *dBuf = (struct _TTMBuffer *)calloc(1, sizeof(*dBuf)); |
| struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); |
| union ttm_pl_reference_arg arg; |
| int ret; |
| |
| if (!dBuf) |
| return NULL; |
| |
| ret = wsbmBufStorageInit(&dBuf->buf, pool); |
| if (ret) |
| goto out_err0; |
| |
| ret = WSBM_COND_INIT(&dBuf->event); |
| if (ret) |
| goto out_err1; |
| |
| arg.req.handle = handle; |
| ret = drmCommandWriteRead(pool->fd, ttmPool->devOffset + TTM_PL_REFERENCE, |
| &arg, sizeof(arg)); |
| |
| if (ret) |
| goto out_err2; |
| |
| dBuf->requestedSize = arg.rep.bo_size; |
| dBuf->kBuf.gpuOffset = arg.rep.gpu_offset; |
| dBuf->mapHandle = arg.rep.map_handle; |
| dBuf->realSize = arg.rep.bo_size; |
| dBuf->kBuf.placement = arg.rep.placement; |
| dBuf->kBuf.handle = arg.rep.handle; |
| dBuf->kBuf.fence_type_mask = arg.rep.sync_object_arg; |
| |
| return &dBuf->buf; |
| |
| out_err2: |
| WSBM_COND_FREE(&dBuf->event); |
| out_err1: |
| wsbmBufStorageTakedown(&dBuf->buf); |
| out_err0: |
| free(dBuf); |
| return NULL; |
| } |
| |
| static void |
| pool_destroy(struct _WsbmBufStorage **buf) |
| { |
| struct _TTMBuffer *dBuf = ttmBuffer(*buf); |
| struct _TTMPool *ttmPool = ttmGetPool(dBuf); |
| struct ttm_pl_reference_req arg; |
| |
| *buf = NULL; |
| if (dBuf->virtual != NULL) { |
| (void)munmap(dBuf->virtual, dBuf->requestedSize); |
| dBuf->virtual = NULL; |
| } |
| arg.handle = dBuf->kBuf.handle; |
| (void)drmCommandWrite(dBuf->buf.pool->fd, |
| ttmPool->devOffset + TTM_PL_UNREF, |
| &arg, sizeof(arg)); |
| |
| WSBM_COND_FREE(&dBuf->event); |
| wsbmBufStorageTakedown(&dBuf->buf); |
| free(dBuf); |
| } |
| |
| static int |
| syncforcpu_locked(struct _WsbmBufStorage *buf, unsigned mode) |
| { |
| uint32_t kmode = 0; |
| struct _TTMBuffer *dBuf = ttmBuffer(buf); |
| struct _TTMPool *ttmPool = ttmGetPool(dBuf); |
| unsigned int readers; |
| unsigned int writers; |
| int ret = 0; |
| |
| while (dBuf->syncInProgress) |
| WSBM_COND_WAIT(&dBuf->event, &buf->mutex); |
| |
| readers = dBuf->readers; |
| writers = dBuf->writers; |
| |
| if ((mode & WSBM_SYNCCPU_READ) && (++dBuf->readers == 1)) |
| kmode |= TTM_PL_SYNCCPU_MODE_READ; |
| |
| if ((mode & WSBM_SYNCCPU_WRITE) && (++dBuf->writers == 1)) |
| kmode |= TTM_PL_SYNCCPU_MODE_WRITE; |
| |
| if (kmode) { |
| struct ttm_pl_synccpu_arg arg; |
| |
| if (mode & WSBM_SYNCCPU_DONT_BLOCK) |
| kmode |= TTM_PL_SYNCCPU_MODE_NO_BLOCK; |
| |
| dBuf->syncInProgress = 1; |
| |
| /* |
| * This might be a lengthy wait, so |
| * release the mutex. |
| */ |
| |
| WSBM_MUTEX_UNLOCK(&buf->mutex); |
| |
| arg.handle = dBuf->kBuf.handle; |
| arg.access_mode = kmode; |
| arg.op = TTM_PL_SYNCCPU_OP_GRAB; |
| |
| DRMRESTARTCOMMANDWRITE(dBuf->buf.pool->fd, |
| ttmPool->devOffset + TTM_PL_SYNCCPU, arg, ret); |
| |
| WSBM_MUTEX_LOCK(&buf->mutex); |
| dBuf->syncInProgress = 0; |
| WSBM_COND_BROADCAST(&dBuf->event); |
| |
| if (ret) { |
| dBuf->readers = readers; |
| dBuf->writers = writers; |
| } |
| } |
| |
| return ret; |
| } |
| |
| static int |
| releasefromcpu_locked(struct _WsbmBufStorage *buf, unsigned mode) |
| { |
| uint32_t kmode = 0; |
| struct _TTMBuffer *dBuf = ttmBuffer(buf); |
| struct _TTMPool *ttmPool = ttmGetPool(dBuf); |
| int ret = 0; |
| |
| while (dBuf->syncInProgress) |
| WSBM_COND_WAIT(&dBuf->event, &buf->mutex); |
| |
| if ((mode & WSBM_SYNCCPU_READ) && (--dBuf->readers == 0)) |
| kmode |= TTM_PL_SYNCCPU_MODE_READ; |
| |
| if ((mode & WSBM_SYNCCPU_WRITE) && (--dBuf->writers == 0)) |
| kmode |= TTM_PL_SYNCCPU_MODE_WRITE; |
| |
| if (kmode) { |
| struct ttm_pl_synccpu_arg arg; |
| |
| arg.handle = dBuf->kBuf.handle; |
| arg.access_mode = kmode; |
| arg.op = TTM_PL_SYNCCPU_OP_RELEASE; |
| |
| DRMRESTARTCOMMANDWRITE(dBuf->buf.pool->fd, |
| ttmPool->devOffset + TTM_PL_SYNCCPU, arg, ret); |
| |
| } |
| |
| return ret; |
| } |
| |
| static int |
| pool_syncforcpu(struct _WsbmBufStorage *buf, unsigned mode) |
| { |
| int ret; |
| |
| WSBM_MUTEX_LOCK(&buf->mutex); |
| ret = syncforcpu_locked(buf, mode); |
| WSBM_MUTEX_UNLOCK(&buf->mutex); |
| return ret; |
| } |
| |
| static void |
| pool_releasefromcpu(struct _WsbmBufStorage *buf, unsigned mode) |
| { |
| WSBM_MUTEX_LOCK(&buf->mutex); |
| (void)releasefromcpu_locked(buf, mode); |
| WSBM_MUTEX_UNLOCK(&buf->mutex); |
| } |
| |
| #ifdef ANDROID |
| |
| /* No header but syscall provided by bionic */ |
| void* __mmap2(void*, size_t, int, int, int, size_t); |
| #define MMAP2_SHIFT 12 // 2**12 == 4096 |
| |
| static void* _temp_mmap(void *addr, size_t size, int prot, int flags, int fd, long long offset) |
| { |
| return __mmap2(addr, size, prot, flags, fd, (unsigned long)(offset >> MMAP2_SHIFT)); |
| } |
| |
| #endif |
| |
| static int |
| pool_map(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)), void **virtual) |
| { |
| struct _TTMBuffer *dBuf = ttmBuffer(buf); |
| void *virt; |
| int ret = 0; |
| |
| WSBM_MUTEX_LOCK(&buf->mutex); |
| |
| /* |
| * mmaps are expensive, so we only really unmap if |
| * we destroy the buffer. |
| */ |
| |
| if (dBuf->virtual == NULL) { |
| #if defined(__LP64__) || defined(_LP64) || defined(__LP64) |
| virt = mmap(0, dBuf->requestedSize, |
| PROT_READ | PROT_WRITE, MAP_SHARED, |
| buf->pool->fd, dBuf->mapHandle); |
| #else |
| virt = _temp_mmap(0, dBuf->requestedSize, |
| PROT_READ | PROT_WRITE, MAP_SHARED, |
| buf->pool->fd, dBuf->mapHandle); |
| #endif |
| if (virt == MAP_FAILED) { |
| ret = -errno; |
| goto out_unlock; |
| } |
| dBuf->virtual = virt; |
| } |
| |
| *virtual = dBuf->virtual; |
| out_unlock: |
| |
| WSBM_MUTEX_UNLOCK(&buf->mutex); |
| |
| return ret; |
| } |
| |
| static void |
| pool_unmap(struct _WsbmBufStorage *buf __attribute__ ((unused))) |
| { |
| ; |
| } |
| |
| static unsigned long |
| pool_offset(struct _WsbmBufStorage *buf) |
| { |
| struct _TTMBuffer *dBuf = ttmBuffer(buf); |
| |
| return dBuf->kBuf.gpuOffset; |
| } |
| |
| static unsigned long |
| pool_poolOffset(struct _WsbmBufStorage *buf __attribute__ ((unused))) |
| { |
| return 0; |
| } |
| |
| static uint32_t |
| pool_placement(struct _WsbmBufStorage *buf) |
| { |
| struct _TTMBuffer *dBuf = ttmBuffer(buf); |
| |
| return dBuf->kBuf.placement; |
| } |
| |
| static unsigned long |
| pool_size(struct _WsbmBufStorage *buf) |
| { |
| struct _TTMBuffer *dBuf = ttmBuffer(buf); |
| |
| return dBuf->realSize; |
| } |
| |
| static void |
| pool_fence(struct _WsbmBufStorage *buf __attribute__ ((unused)), |
| struct _WsbmFenceObject *fence __attribute__ ((unused))) |
| { |
| /* |
| * Noop. The kernel handles all fencing. |
| */ |
| } |
| |
| static int |
| pool_waitIdle(struct _WsbmBufStorage *buf, int lazy) |
| { |
| struct _TTMBuffer *dBuf = ttmBuffer(buf); |
| struct _TTMPool *ttmPool = ttmGetPool(dBuf); |
| struct ttm_pl_waitidle_arg req; |
| struct _WsbmBufferPool *pool = buf->pool; |
| int ret; |
| |
| req.handle = dBuf->kBuf.handle; |
| req.mode = (lazy) ? TTM_PL_WAITIDLE_MODE_LAZY : 0; |
| |
| DRMRESTARTCOMMANDWRITE(pool->fd, ttmPool->devOffset + TTM_PL_WAITIDLE, |
| req, ret); |
| |
| return ret; |
| } |
| |
| static void |
| pool_takedown(struct _WsbmBufferPool *pool) |
| { |
| struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); |
| |
| free(ttmPool); |
| } |
| |
| static int |
| pool_setStatus(struct _WsbmBufStorage *buf, uint32_t set_placement, |
| uint32_t clr_placement) |
| { |
| struct _TTMBuffer *dBuf = ttmBuffer(buf); |
| struct _TTMPool *ttmPool = ttmGetPool(dBuf); |
| union ttm_pl_setstatus_arg arg; |
| struct ttm_pl_setstatus_req *req = &arg.req; |
| struct ttm_pl_rep *rep = &arg.rep; |
| struct _WsbmBufferPool *pool = buf->pool; |
| int ret; |
| |
| req->handle = dBuf->kBuf.handle; |
| req->set_placement = set_placement; |
| req->clr_placement = clr_placement; |
| |
| DRMRESTARTCOMMANDWRITEREAD(pool->fd, |
| ttmPool->devOffset + TTM_PL_SETSTATUS, |
| arg, ret); |
| |
| if (!ret) { |
| dBuf->kBuf.gpuOffset = rep->gpu_offset; |
| dBuf->kBuf.placement = rep->placement; |
| } |
| |
| return ret; |
| } |
| |
| static struct _WsbmKernelBuf * |
| pool_kernel(struct _WsbmBufStorage *buf) |
| { |
| return (void *)&ttmBuffer(buf)->kBuf; |
| } |
| |
| struct _WsbmBufferPool * |
| wsbmTTMPoolInit(int fd, unsigned int devOffset) |
| { |
| struct _TTMPool *ttmPool; |
| struct _WsbmBufferPool *pool; |
| |
| ttmPool = (struct _TTMPool *)calloc(1, sizeof(*ttmPool)); |
| |
| if (!ttmPool) |
| return NULL; |
| |
| ttmPool->pageSize = getpagesize(); |
| ttmPool->devOffset = devOffset; |
| pool = &ttmPool->pool; |
| |
| pool->fd = fd; |
| pool->map = &pool_map; |
| pool->unmap = &pool_unmap; |
| pool->syncforcpu = &pool_syncforcpu; |
| pool->releasefromcpu = &pool_releasefromcpu; |
| pool->destroy = &pool_destroy; |
| pool->offset = &pool_offset; |
| pool->poolOffset = &pool_poolOffset; |
| pool->placement = &pool_placement; |
| pool->size = &pool_size; |
| pool->create = &pool_create; |
| pool->fence = &pool_fence; |
| pool->kernel = &pool_kernel; |
| pool->validate = NULL; |
| pool->unvalidate = NULL; |
| pool->waitIdle = &pool_waitIdle; |
| pool->takeDown = &pool_takedown; |
| pool->createByReference = &pool_reference; |
| pool->setStatus = &pool_setStatus; |
| return pool; |
| } |
| |
| struct _WsbmBufStorage * |
| ttm_pool_ub_create(struct _WsbmBufferPool *pool, unsigned long size, uint32_t placement, unsigned alignment, const unsigned long *user_ptr, int fd) |
| { |
| struct _TTMBuffer *dBuf = (struct _TTMBuffer *) |
| calloc(1, sizeof(*dBuf)); |
| struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); |
| int ret; |
| unsigned pageSize = ttmPool->pageSize; |
| union ttm_pl_create_ub_arg arg; |
| |
| if (!dBuf) |
| return NULL; |
| |
| if ((alignment > pageSize) && (alignment % pageSize)) |
| goto out_err0; |
| |
| ret = wsbmBufStorageInit(&dBuf->buf, pool); |
| if (ret) |
| goto out_err0; |
| |
| ret = WSBM_COND_INIT(&dBuf->event); |
| if (ret) |
| goto out_err1; |
| |
| arg.req.size = size; |
| arg.req.placement = placement; |
| arg.req.page_alignment = alignment / pageSize; |
| arg.req.user_address = (unsigned long)user_ptr; |
| arg.req.fd = fd; |
| |
| DRMRESTARTCOMMANDWRITEREAD(pool->fd, ttmPool->devOffset + TTM_PL_CREATE_UB, |
| arg, ret); |
| if (ret) |
| goto out_err2; |
| |
| dBuf->requestedSize = size; |
| dBuf->kBuf.gpuOffset = arg.rep.gpu_offset; |
| dBuf->mapHandle = arg.rep.map_handle; |
| dBuf->realSize = arg.rep.bo_size; |
| dBuf->kBuf.placement = arg.rep.placement; |
| dBuf->kBuf.handle = arg.rep.handle; |
| |
| return &dBuf->buf; |
| |
| out_err2: |
| WSBM_COND_FREE(&dBuf->event); |
| out_err1: |
| wsbmBufStorageTakedown(&dBuf->buf); |
| out_err0: |
| free(dBuf); |
| return NULL; |
| } |
| |