src/wsbm_fencemgr.c

/**************************************************************************
 *
 * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, Tx., USA
 * All Rights Reserved.
 * Copyright 2009 VMware, Inc., Palo Alto, CA., USA
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 **************************************************************************/
/*
 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include "wsbm_fencemgr.h"
#include "wsbm_pool.h"
#include "wsbm_manager.h"
#include <xf86drm.h>
#include <drm/psb_ttm_fence_user.h>
#include <string.h>
#include <unistd.h>

struct _WsbmFenceClass
{
    struct _WsbmListHead head;
    struct _WsbmMutex mutex;
    struct _WsbmMutex cmd_mutex;
};

/*
 * Note: The struct _WsbmFenceMgr::Mutex should never be held
 * during sleeps, since that may block fast concurrent access to
 * fence data.
 */

struct _WsbmFenceMgr
{
    /*
     * Constant members. Need no mutex protection.
     */
    struct _WsbmFenceMgrCreateInfo info;
    void *private;

    /*
     * Atomic members. No mutex protection.
     */

    struct _WsbmAtomic count;

    /*
     * These members are protected by this->mutex
     */

    struct _WsbmFenceClass *classes;
    uint32_t num_classes;
};

struct _WsbmFenceObject
{

    /*
     * These members are constant and need no mutex protection.
     * Note that @private may point to a structure with its own
     * mutex protection, that we don't care about.
     */

    struct _WsbmFenceMgr *mgr;
    uint32_t fence_class;
    uint32_t fence_type;
    void *private;

    /*
     * Atomic members. No mutex protection. note that
     * @signaled types is updated using a compare-and-swap
     * scheme to guarantee atomicity.
     */

    struct _WsbmAtomic refCount;
    struct _WsbmAtomic signaled_types;

    /*
     * These members are protected by mgr->mutex.
     */
    struct _WsbmListHead head;
};

uint32_t
wsbmFenceType(struct _WsbmFenceObject *fence)
{
    return fence->fence_type;
}

struct _WsbmFenceMgr *
wsbmFenceMgrCreate(const struct _WsbmFenceMgrCreateInfo *info)
{
    struct _WsbmFenceMgr *tmp;
    uint32_t i, j;
    int ret;

    tmp = calloc(1, sizeof(*tmp));
    if (!tmp)
	return NULL;

    tmp->info = *info;
    tmp->classes = calloc(tmp->info.num_classes, sizeof(*tmp->classes));
    if (!tmp->classes)
	goto out_err;

    for (i = 0; i < tmp->info.num_classes; ++i) {
	struct _WsbmFenceClass *fc = &tmp->classes[i];

	WSBMINITLISTHEAD(&fc->head);
	ret = WSBM_MUTEX_INIT(&fc->mutex);
	if (ret)
	    goto out_err1;
	ret = WSBM_MUTEX_INIT(&fc->cmd_mutex);
	if (ret) {
	    WSBM_MUTEX_FREE(&fc->mutex);
	    goto out_err1;
	}
    }
    wsbmAtomicSet(&tmp->count, 0);

    return tmp;

  out_err1:
    for (j = 0; j < i; ++j) {
	WSBM_MUTEX_FREE(&tmp->classes[j].mutex);
	WSBM_MUTEX_FREE(&tmp->classes[j].cmd_mutex);
    }
    free(tmp->classes);
  out_err:
    if (tmp)
	free(tmp);
    return NULL;
}

void
wsbmFenceUnreference(struct _WsbmFenceObject **pFence)
{
    struct _WsbmFenceObject *fence = *pFence;
    struct _WsbmFenceMgr *mgr;

    *pFence = NULL;
    if (fence == NULL)
	return;

    mgr = fence->mgr;
    if (wsbmAtomicDecZero(&fence->refCount)) {
	struct _WsbmFenceClass *fc = &mgr->classes[fence->fence_class];

	WSBM_MUTEX_LOCK(&fc->mutex);
	WSBMLISTDELINIT(&fence->head);
	WSBM_MUTEX_UNLOCK(&fc->mutex);
	if (fence->private)
	    mgr->info.unreference(mgr, &fence->private);
	fence->mgr = NULL;
	wsbmAtomicDecZero(&mgr->count);
	free(fence);
    }
}

static void
wsbmSignalPreviousFences(struct _WsbmFenceMgr *mgr,
			 struct _WsbmListHead *list,
			 uint32_t fence_class, uint32_t signaled_types)
{
    struct _WsbmFenceClass *fc = &mgr->classes[fence_class];
    struct _WsbmFenceObject *entry;
    struct _WsbmListHead *prev;
    uint32_t old_signaled_types;
    uint32_t ret_st;

    WSBM_MUTEX_LOCK(&fc->mutex);
    while (list != &fc->head && list->next != list) {
	entry = WSBMLISTENTRY(list, struct _WsbmFenceObject, head);

	prev = list->prev;

	do {
	    old_signaled_types = wsbmAtomicRead(&entry->signaled_types);
	    signaled_types =
		old_signaled_types | (signaled_types & entry->fence_type);
	    if (signaled_types == old_signaled_types)
		break;

	    ret_st =
		wsbmAtomicCmpXchg(&entry->signaled_types, old_signaled_types,
				  signaled_types);
	} while (ret_st != old_signaled_types);

	if (signaled_types == entry->fence_type)
	    WSBMLISTDELINIT(list);

	list = prev;
    }
    WSBM_MUTEX_UNLOCK(&fc->mutex);
}

int
wsbmFenceFinish(struct _WsbmFenceObject *fence, uint32_t fence_type,
		int lazy_hint)
{
    struct _WsbmFenceMgr *mgr = fence->mgr;
    int ret = 0;

    if ((wsbmAtomicRead(&fence->signaled_types) & fence_type) == fence_type)
	goto out;

    ret = mgr->info.finish(mgr, fence->private, fence_type, lazy_hint);
    if (ret)
	goto out;

    wsbmSignalPreviousFences(mgr, &fence->head, fence->fence_class,
			     fence_type);
  out:
    return ret;
}

uint32_t
wsbmFenceSignaledTypeCached(struct _WsbmFenceObject * fence)
{
    return wsbmAtomicRead(&fence->signaled_types);
}

int
wsbmFenceSignaledType(struct _WsbmFenceObject *fence, uint32_t flush_type,
		      uint32_t * signaled)
{
    int ret = 0;
    struct _WsbmFenceMgr *mgr;
    uint32_t signaled_types;
    uint32_t old_signaled_types;
    uint32_t ret_st;

    mgr = fence->mgr;
    *signaled = wsbmAtomicRead(&fence->signaled_types);
    if ((*signaled & flush_type) == flush_type)
	goto out0;

    ret = mgr->info.signaled(mgr, fence->private, flush_type, signaled);
    if (ret) {
	*signaled = wsbmAtomicRead(&fence->signaled_types);
	goto out0;
    }

    do {
	old_signaled_types = wsbmAtomicRead(&fence->signaled_types);
	signaled_types = old_signaled_types | *signaled;
	if (signaled_types == old_signaled_types)
	    break;

	ret_st = wsbmAtomicCmpXchg(&fence->signaled_types, old_signaled_types,
				   signaled_types);
	if (old_signaled_types == ret_st)
	    wsbmSignalPreviousFences(mgr, &fence->head, fence->fence_class,
				     *signaled);
    } while (old_signaled_types != ret_st);

    return 0;
  out0:
    return ret;
}

struct _WsbmFenceObject *
wsbmFenceReference(struct _WsbmFenceObject *fence)
{
    if (fence == NULL)
	return NULL;
    wsbmAtomicInc(&fence->refCount);
    return fence;
}

struct _WsbmFenceObject *
wsbmFenceCreateSig(struct _WsbmFenceMgr *mgr, uint32_t fence_class,
		   uint32_t fence_type, uint32_t signaled_types, 
		   void *private, size_t private_size)
{
    struct _WsbmFenceClass *fc = &mgr->classes[fence_class];
    struct _WsbmFenceObject *fence;
    size_t fence_size = sizeof(*fence);

    if (private_size)
	fence_size = ((fence_size + 15) & ~15);

    fence = calloc(1, fence_size + private_size);

    if (!fence)
	goto out_err;

    wsbmAtomicSet(&fence->refCount, 1);
    fence->mgr = mgr;
    fence->fence_class = fence_class;
    fence->fence_type = fence_type;
    wsbmAtomicSet(&fence->signaled_types, signaled_types);
    fence->private = private;
    if (private_size) {
	fence->private = (void *)(((uint8_t *) fence) + fence_size);
	memcpy(fence->private, private, private_size);
    }

    WSBM_MUTEX_LOCK(&fc->mutex);
    WSBMLISTADDTAIL(&fence->head, &fc->head);
    WSBM_MUTEX_UNLOCK(&fc->mutex);
    wsbmAtomicInc(&mgr->count);
    return fence;

  out_err:
    {
	int ret = mgr->info.finish(mgr, private, fence_type, 0);

	if (ret)
	    usleep(10000000);
    }
    if (fence)
	free(fence);

    mgr->info.unreference(mgr, &private);
    return NULL;
}

struct _WsbmFenceObject *
wsbmFenceCreate(struct _WsbmFenceMgr *mgr, uint32_t fence_class,
		uint32_t fence_type, void *private, size_t private_size)
{
  return wsbmFenceCreateSig(mgr, fence_class, fence_type, 0, private,
			    private_size);
}

struct _WsbmTTMFenceMgrPriv
{
    int fd;
    unsigned int devOffset;
};

static int
tSignaled(struct _WsbmFenceMgr *mgr, void *private, uint32_t flush_type,
	  uint32_t * signaled_type)
{
    struct _WsbmTTMFenceMgrPriv *priv =
	(struct _WsbmTTMFenceMgrPriv *)mgr->private;
    union ttm_fence_signaled_arg arg;
    int ret;

    arg.req.handle = (unsigned long)private;
    arg.req.fence_type = flush_type;
    arg.req.flush = 1;
    *signaled_type = 0;

    ret = drmCommandWriteRead(priv->fd, priv->devOffset + TTM_FENCE_SIGNALED,
			      &arg, sizeof(arg));
    if (ret)
	return ret;

    *signaled_type = arg.rep.signaled_types;
    return 0;
}

static int
tFinish(struct _WsbmFenceMgr *mgr, void *private, uint32_t fence_type,
	int lazy_hint)
{
    struct _WsbmTTMFenceMgrPriv *priv =
	(struct _WsbmTTMFenceMgrPriv *)mgr->private;
    union ttm_fence_finish_arg arg =
	{.req = {.handle = (unsigned long)private,
		 .fence_type = fence_type,
		 .mode = (lazy_hint) ? TTM_FENCE_FINISH_MODE_LAZY : 0}
    };
    int ret;

    do {
	ret = drmCommandWriteRead(priv->fd, priv->devOffset + TTM_FENCE_FINISH,
				  &arg, sizeof(arg));
    } while (ret == -EAGAIN || ret == -ERESTART);

    return ret;
}

static int
tUnref(struct _WsbmFenceMgr *mgr, void **private)
{
    struct _WsbmTTMFenceMgrPriv *priv =
	(struct _WsbmTTMFenceMgrPriv *)mgr->private;
    struct ttm_fence_unref_arg arg = {.handle = (unsigned long)*private };

    *private = NULL;

    return drmCommandWrite(priv->fd, priv->devOffset + TTM_FENCE_UNREF,
			   &arg, sizeof(arg));
}

struct _WsbmFenceMgr *
wsbmFenceMgrTTMInit(int fd, unsigned int numClass, unsigned int devOffset)
{
    struct _WsbmFenceMgrCreateInfo info;
    struct _WsbmFenceMgr *mgr;
    struct _WsbmTTMFenceMgrPriv *priv = malloc(sizeof(*priv));

    if (!priv)
	return NULL;

    priv->fd = fd;
    priv->devOffset = devOffset;

    info.flags = WSBM_FENCE_CLASS_ORDERED;
    info.num_classes = numClass;
    info.signaled = tSignaled;
    info.finish = tFinish;
    info.unreference = tUnref;

    mgr = wsbmFenceMgrCreate(&info);
    if (mgr == NULL) {
	free(priv);
	return NULL;
    }

    mgr->private = (void *)priv;
    return mgr;
}

void
wsbmFenceCmdLock(struct _WsbmFenceMgr *mgr, uint32_t fence_class)
{
    WSBM_MUTEX_LOCK(&mgr->classes[fence_class].cmd_mutex);
}

void
wsbmFenceCmdUnlock(struct _WsbmFenceMgr *mgr, uint32_t fence_class)
{
    WSBM_MUTEX_UNLOCK(&mgr->classes[fence_class].cmd_mutex);
}

void
wsbmFenceMgrTTMTakedown(struct _WsbmFenceMgr *mgr)
{
    unsigned int i;

    if (!mgr)
	return;

    if (mgr->private)
	free(mgr->private);

    for (i = 0; i < mgr->info.num_classes; ++i) {
	WSBM_MUTEX_FREE(&mgr->classes[i].mutex);
	WSBM_MUTEX_FREE(&mgr->classes[i].cmd_mutex);
    }
    free(mgr);

    return;
}