// Copyright (c) 2005, 2006, Google Inc. | |
// All rights reserved. | |
// | |
// Redistribution and use in source and binary forms, with or without | |
// modification, are permitted provided that the following conditions are | |
// met: | |
// | |
// * Redistributions of source code must retain the above copyright | |
// notice, this list of conditions and the following disclaimer. | |
// * Redistributions in binary form must reproduce the above | |
// copyright notice, this list of conditions and the following disclaimer | |
// in the documentation and/or other materials provided with the | |
// distribution. | |
// * Neither the name of Google Inc. nor the names of its | |
// contributors may be used to endorse or promote products derived from | |
// this software without specific prior written permission. | |
// | |
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
// --- | |
// Author: Sanjay Ghemawat <opensource@google.com> | |
#ifndef TCMALLOC_INTERNAL_SPINLOCK_H__ | |
#define TCMALLOC_INTERNAL_SPINLOCK_H__ | |
#if (CPU(X86) || CPU(X86_64) || CPU(PPC)) && (COMPILER(GCC) || COMPILER(MSVC)) | |
#include <time.h> /* For nanosleep() */ | |
#include <sched.h> /* For sched_yield() */ | |
#if HAVE(STDINT_H) | |
#include <stdint.h> | |
#elif HAVE(INTTYPES_H) | |
#include <inttypes.h> | |
#else | |
#include <sys/types.h> | |
#endif | |
#if OS(WINDOWS) | |
#ifndef WIN32_LEAN_AND_MEAN | |
#define WIN32_LEAN_AND_MEAN | |
#endif | |
#include <windows.h> | |
#endif | |
static void TCMalloc_SlowLock(volatile unsigned int* lockword); | |
// The following is a struct so that it can be initialized at compile time | |
struct TCMalloc_SpinLock { | |
inline void Lock() { | |
int r; | |
#if COMPILER(GCC) | |
#if CPU(X86) || CPU(X86_64) | |
__asm__ __volatile__ | |
("xchgl %0, %1" | |
: "=r"(r), "=m"(lockword_) | |
: "0"(1), "m"(lockword_) | |
: "memory"); | |
#else | |
volatile unsigned int *lockword_ptr = &lockword_; | |
__asm__ __volatile__ | |
("1: lwarx %0, 0, %1\n\t" | |
"stwcx. %2, 0, %1\n\t" | |
"bne- 1b\n\t" | |
"isync" | |
: "=&r" (r), "=r" (lockword_ptr) | |
: "r" (1), "1" (lockword_ptr) | |
: "memory"); | |
#endif | |
#elif COMPILER(MSVC) | |
__asm { | |
mov eax, this ; store &lockword_ (which is this+0) in eax | |
mov ebx, 1 ; store 1 in ebx | |
xchg [eax], ebx ; exchange lockword_ and 1 | |
mov r, ebx ; store old value of lockword_ in r | |
} | |
#endif | |
if (r) TCMalloc_SlowLock(&lockword_); | |
} | |
inline void Unlock() { | |
#if COMPILER(GCC) | |
#if CPU(X86) || CPU(X86_64) | |
__asm__ __volatile__ | |
("movl $0, %0" | |
: "=m"(lockword_) | |
: "m" (lockword_) | |
: "memory"); | |
#else | |
__asm__ __volatile__ | |
("isync\n\t" | |
"eieio\n\t" | |
"stw %1, %0" | |
#if OS(DARWIN) || CPU(PPC) | |
: "=o" (lockword_) | |
#else | |
: "=m" (lockword_) | |
#endif | |
: "r" (0) | |
: "memory"); | |
#endif | |
#elif COMPILER(MSVC) | |
__asm { | |
mov eax, this ; store &lockword_ (which is this+0) in eax | |
mov [eax], 0 ; set lockword_ to 0 | |
} | |
#endif | |
} | |
// Report if we think the lock can be held by this thread. | |
// When the lock is truly held by the invoking thread | |
// we will always return true. | |
// Indended to be used as CHECK(lock.IsHeld()); | |
inline bool IsHeld() const { | |
return lockword_ != 0; | |
} | |
inline void Init() { lockword_ = 0; } | |
volatile unsigned int lockword_; | |
}; | |
#define SPINLOCK_INITIALIZER { 0 } | |
static void TCMalloc_SlowLock(volatile unsigned int* lockword) { | |
sched_yield(); // Yield immediately since fast path failed | |
while (true) { | |
int r; | |
#if COMPILER(GCC) | |
#if CPU(X86) || CPU(X86_64) | |
__asm__ __volatile__ | |
("xchgl %0, %1" | |
: "=r"(r), "=m"(*lockword) | |
: "0"(1), "m"(*lockword) | |
: "memory"); | |
#else | |
int tmp = 1; | |
__asm__ __volatile__ | |
("1: lwarx %0, 0, %1\n\t" | |
"stwcx. %2, 0, %1\n\t" | |
"bne- 1b\n\t" | |
"isync" | |
: "=&r" (r), "=r" (lockword) | |
: "r" (tmp), "1" (lockword) | |
: "memory"); | |
#endif | |
#elif COMPILER(MSVC) | |
__asm { | |
mov eax, lockword ; assign lockword into eax | |
mov ebx, 1 ; assign 1 into ebx | |
xchg [eax], ebx ; exchange *lockword and 1 | |
mov r, ebx ; store old value of *lockword in r | |
} | |
#endif | |
if (!r) { | |
return; | |
} | |
// This code was adapted from the ptmalloc2 implementation of | |
// spinlocks which would sched_yield() upto 50 times before | |
// sleeping once for a few milliseconds. Mike Burrows suggested | |
// just doing one sched_yield() outside the loop and always | |
// sleeping after that. This change helped a great deal on the | |
// performance of spinlocks under high contention. A test program | |
// with 10 threads on a dual Xeon (four virtual processors) went | |
// from taking 30 seconds to 16 seconds. | |
// Sleep for a few milliseconds | |
#if OS(WINDOWS) | |
Sleep(2); | |
#else | |
struct timespec tm; | |
tm.tv_sec = 0; | |
tm.tv_nsec = 2000001; | |
nanosleep(&tm, NULL); | |
#endif | |
} | |
} | |
#else | |
#include <pthread.h> | |
// Portable version | |
struct TCMalloc_SpinLock { | |
pthread_mutex_t private_lock_; | |
inline void Init() { | |
if (pthread_mutex_init(&private_lock_, NULL) != 0) CRASH(); | |
} | |
inline void Finalize() { | |
if (pthread_mutex_destroy(&private_lock_) != 0) CRASH(); | |
} | |
inline void Lock() { | |
if (pthread_mutex_lock(&private_lock_) != 0) CRASH(); | |
} | |
inline void Unlock() { | |
if (pthread_mutex_unlock(&private_lock_) != 0) CRASH(); | |
} | |
bool IsHeld() { | |
if (pthread_mutex_trylock(&private_lock_)) | |
return true; | |
Unlock(); | |
return false; | |
} | |
}; | |
#define SPINLOCK_INITIALIZER { PTHREAD_MUTEX_INITIALIZER } | |
#endif | |
// Corresponding locker object that arranges to acquire a spinlock for | |
// the duration of a C++ scope. | |
class TCMalloc_SpinLockHolder { | |
private: | |
TCMalloc_SpinLock* lock_; | |
public: | |
inline explicit TCMalloc_SpinLockHolder(TCMalloc_SpinLock* l) | |
: lock_(l) { l->Lock(); } | |
inline ~TCMalloc_SpinLockHolder() { lock_->Unlock(); } | |
}; | |
// Short-hands for convenient use by tcmalloc.cc | |
typedef TCMalloc_SpinLock SpinLock; | |
typedef TCMalloc_SpinLockHolder SpinLockHolder; | |
#endif // TCMALLOC_INTERNAL_SPINLOCK_H__ |