blob: 5ddafb4b89a14f90eea8d8a3346ee3e6a75bfa78 [file] [log] [blame]
/*
* Copyright (C) 2008 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef ART_RUNTIME_ATOMIC_H_
#define ART_RUNTIME_ATOMIC_H_
#ifdef __clang__
#define ART_HAVE_STDATOMIC 1
#endif
#include <stdint.h>
#if ART_HAVE_STDATOMIC
#include <atomic>
#endif
#include <limits>
#include <vector>
#include "base/logging.h"
#include "base/macros.h"
namespace art {
class Mutex;
// QuasiAtomic encapsulates two separate facilities that we are
// trying to move away from: "quasiatomic" 64 bit operations
// and custom memory fences. For the time being, they remain
// exposed. Clients should be converted to use either class Atomic
// below whenever possible, and should eventually use C++11 atomics.
// The two facilities that do not have a good C++11 analog are
// ThreadFenceForConstructor and Atomic::*JavaData.
//
// NOTE: Two "quasiatomic" operations on the exact same memory address
// are guaranteed to operate atomically with respect to each other,
// but no guarantees are made about quasiatomic operations mixed with
// non-quasiatomic operations on the same address, nor about
// quasiatomic operations that are performed on partially-overlapping
// memory.
class QuasiAtomic {
#if defined(__mips__) && !defined(__LP64__)
static constexpr bool kNeedSwapMutexes = true;
#else
static constexpr bool kNeedSwapMutexes = false;
#endif
public:
static void Startup();
static void Shutdown();
// Reads the 64-bit value at "addr" without tearing.
static int64_t Read64(volatile const int64_t* addr) {
if (!kNeedSwapMutexes) {
int64_t value;
#if defined(__LP64__)
value = *addr;
#else
#if defined(__arm__)
#if defined(__ARM_FEATURE_LPAE)
// With LPAE support (such as Cortex-A15) then ldrd is defined not to tear.
__asm__ __volatile__("@ QuasiAtomic::Read64\n"
"ldrd %0, %H0, %1"
: "=r" (value)
: "m" (*addr));
#else
// Exclusive loads are defined not to tear, clearing the exclusive state isn't necessary.
__asm__ __volatile__("@ QuasiAtomic::Read64\n"
"ldrexd %0, %H0, %1"
: "=r" (value)
: "Q" (*addr));
#endif
#elif defined(__i386__)
__asm__ __volatile__(
"movq %1, %0\n"
: "=x" (value)
: "m" (*addr));
#else
LOG(FATAL) << "Unsupported architecture";
#endif
#endif // defined(__LP64__)
return value;
} else {
return SwapMutexRead64(addr);
}
}
// Writes to the 64-bit value at "addr" without tearing.
static void Write64(volatile int64_t* addr, int64_t value) {
if (!kNeedSwapMutexes) {
#if defined(__LP64__)
*addr = value;
#else
#if defined(__arm__)
#if defined(__ARM_FEATURE_LPAE)
// If we know that ARM architecture has LPAE (such as Cortex-A15) strd is defined not to tear.
__asm__ __volatile__("@ QuasiAtomic::Write64\n"
"strd %1, %H1, %0"
: "=m"(*addr)
: "r" (value));
#else
// The write is done as a swap so that the cache-line is in the exclusive state for the store.
int64_t prev;
int status;
do {
__asm__ __volatile__("@ QuasiAtomic::Write64\n"
"ldrexd %0, %H0, %2\n"
"strexd %1, %3, %H3, %2"
: "=&r" (prev), "=&r" (status), "+Q"(*addr)
: "r" (value)
: "cc");
} while (UNLIKELY(status != 0));
#endif
#elif defined(__i386__)
__asm__ __volatile__(
"movq %1, %0"
: "=m" (*addr)
: "x" (value));
#else
LOG(FATAL) << "Unsupported architecture";
#endif
#endif // defined(__LP64__)
} else {
SwapMutexWrite64(addr, value);
}
}
// Atomically compare the value at "addr" to "old_value", if equal replace it with "new_value"
// and return true. Otherwise, don't swap, and return false.
// This is fully ordered, i.e. it has C++11 memory_order_seq_cst
// semantics (assuming all other accesses use a mutex if this one does).
// This has "strong" semantics; if it fails then it is guaranteed that
// at some point during the execution of Cas64, *addr was not equal to
// old_value.
static bool Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) {
if (!kNeedSwapMutexes) {
return __sync_bool_compare_and_swap(addr, old_value, new_value);
} else {
return SwapMutexCas64(old_value, new_value, addr);
}
}
// Does the architecture provide reasonable atomic long operations or do we fall back on mutexes?
static bool LongAtomicsUseMutexes() {
return kNeedSwapMutexes;
}
#if ART_HAVE_STDATOMIC
static void ThreadFenceAcquire() {
std::atomic_thread_fence(std::memory_order_acquire);
}
static void ThreadFenceRelease() {
std::atomic_thread_fence(std::memory_order_release);
}
static void ThreadFenceForConstructor() {
#if defined(__aarch64__)
__asm__ __volatile__("dmb ishst" : : : "memory");
#else
std::atomic_thread_fence(std::memory_order_release);
#endif
}
static void ThreadFenceSequentiallyConsistent() {
std::atomic_thread_fence(std::memory_order_seq_cst);
}
#else
static void ThreadFenceAcquire() {
#if defined(__arm__) || defined(__aarch64__)
__asm__ __volatile__("dmb ish" : : : "memory");
// Could possibly use dmb ishld on aarch64
// But currently we also use this on volatile loads
// to enforce store atomicity. Ishld is
// insufficient for that purpose.
#elif defined(__i386__) || defined(__x86_64__)
__asm__ __volatile__("" : : : "memory");
#elif defined(__mips__)
__asm__ __volatile__("sync" : : : "memory");
#else
#error Unexpected architecture
#endif
}
static void ThreadFenceRelease() {
#if defined(__arm__) || defined(__aarch64__)
__asm__ __volatile__("dmb ish" : : : "memory");
// ishst doesn't order load followed by store.
#elif defined(__i386__) || defined(__x86_64__)
__asm__ __volatile__("" : : : "memory");
#elif defined(__mips__)
__asm__ __volatile__("sync" : : : "memory");
#else
#error Unexpected architecture
#endif
}
// Fence at the end of a constructor with final fields
// or allocation. We believe this
// only has to order stores, and can thus be weaker than
// release on aarch64.
static void ThreadFenceForConstructor() {
#if defined(__arm__) || defined(__aarch64__)
__asm__ __volatile__("dmb ishst" : : : "memory");
#elif defined(__i386__) || defined(__x86_64__)
__asm__ __volatile__("" : : : "memory");
#elif defined(__mips__)
__asm__ __volatile__("sync" : : : "memory");
#else
#error Unexpected architecture
#endif
}
static void ThreadFenceSequentiallyConsistent() {
#if defined(__arm__) || defined(__aarch64__)
__asm__ __volatile__("dmb ish" : : : "memory");
#elif defined(__i386__) || defined(__x86_64__)
__asm__ __volatile__("mfence" : : : "memory");
#elif defined(__mips__)
__asm__ __volatile__("sync" : : : "memory");
#else
#error Unexpected architecture
#endif
}
#endif
private:
static Mutex* GetSwapMutex(const volatile int64_t* addr);
static int64_t SwapMutexRead64(volatile const int64_t* addr);
static void SwapMutexWrite64(volatile int64_t* addr, int64_t val);
static bool SwapMutexCas64(int64_t old_value, int64_t new_value, volatile int64_t* addr);
// We stripe across a bunch of different mutexes to reduce contention.
static constexpr size_t kSwapMutexCount = 32;
static std::vector<Mutex*>* gSwapMutexes;
DISALLOW_COPY_AND_ASSIGN(QuasiAtomic);
};
#if ART_HAVE_STDATOMIC
template<typename T>
class Atomic : public std::atomic<T> {
public:
Atomic<T>() : std::atomic<T>() { }
explicit Atomic<T>(T value) : std::atomic<T>(value) { }
// Load from memory without ordering or synchronization constraints.
T LoadRelaxed() const {
return this->load(std::memory_order_relaxed);
}
// Word tearing allowed, but may race.
// TODO: Optimize?
// There has been some discussion of eventually disallowing word
// tearing for Java data loads.
T LoadJavaData() const {
return this->load(std::memory_order_relaxed);
}
// Load from memory with a total ordering.
// Corresponds exactly to a Java volatile load.
T LoadSequentiallyConsistent() const {
return this->load(std::memory_order_seq_cst);
}
// Store to memory without ordering or synchronization constraints.
void StoreRelaxed(T desired) {
this->store(desired, std::memory_order_relaxed);
}
// Word tearing allowed, but may race.
void StoreJavaData(T desired) {
this->store(desired, std::memory_order_relaxed);
}
// Store to memory with release ordering.
void StoreRelease(T desired) {
this->store(desired, std::memory_order_release);
}
// Store to memory with a total ordering.
void StoreSequentiallyConsistent(T desired) {
this->store(desired, std::memory_order_seq_cst);
}
// Atomically replace the value with desired value if it matches the expected value.
// Participates in total ordering of atomic operations.
bool CompareExchangeStrongSequentiallyConsistent(T expected_value, T desired_value) {
return this->compare_exchange_strong(expected_value, desired_value, std::memory_order_seq_cst);
}
// The same, except it may fail spuriously.
bool CompareExchangeWeakSequentiallyConsistent(T expected_value, T desired_value) {
return this->compare_exchange_weak(expected_value, desired_value, std::memory_order_seq_cst);
}
// Atomically replace the value with desired value if it matches the expected value. Doesn't
// imply ordering or synchronization constraints.
bool CompareExchangeStrongRelaxed(T expected_value, T desired_value) {
return this->compare_exchange_strong(expected_value, desired_value, std::memory_order_relaxed);
}
// The same, except it may fail spuriously.
bool CompareExchangeWeakRelaxed(T expected_value, T desired_value) {
return this->compare_exchange_weak(expected_value, desired_value, std::memory_order_relaxed);
}
// Atomically replace the value with desired value if it matches the expected value. Prior writes
// made to other memory locations by the thread that did the release become visible in this
// thread.
bool CompareExchangeWeakAcquire(T expected_value, T desired_value) {
return this->compare_exchange_weak(expected_value, desired_value, std::memory_order_acquire);
}
// Atomically replace the value with desired value if it matches the expected value. prior writes
// to other memory locations become visible to the threads that do a consume or an acquire on the
// same location.
bool CompareExchangeWeakRelease(T expected_value, T desired_value) {
return this->compare_exchange_weak(expected_value, desired_value, std::memory_order_release);
}
T FetchAndAddSequentiallyConsistent(const T value) {
return this->fetch_add(value, std::memory_order_seq_cst); // Return old_value.
}
T FetchAndSubSequentiallyConsistent(const T value) {
return this->fetch_sub(value, std::memory_order_seq_cst); // Return old value.
}
T FetchAndOrSequentiallyConsistent(const T value) {
return this->fetch_or(value, std::memory_order_seq_cst); // Return old_value.
}
T FetchAndAndSequentiallyConsistent(const T value) {
return this->fetch_and(value, std::memory_order_seq_cst); // Return old_value.
}
volatile T* Address() {
return reinterpret_cast<T*>(this);
}
static T MaxValue() {
return std::numeric_limits<T>::max();
}
};
#else
template<typename T> class Atomic;
// Helper class for Atomic to deal separately with size 8 and small
// objects. Should not be used directly.
template<int SZ, class T> struct AtomicHelper {
friend class Atomic<T>;
private:
COMPILE_ASSERT(sizeof(T) <= 4, bad_atomic_helper_arg);
static T LoadRelaxed(const volatile T* loc) {
// sizeof(T) <= 4
return *loc;
}
static void StoreRelaxed(volatile T* loc, T desired) {
// sizeof(T) <= 4
*loc = desired;
}
static bool CompareExchangeStrongSequentiallyConsistent(volatile T* loc,
T expected_value, T desired_value) {
// sizeof(T) <= 4
return __sync_bool_compare_and_swap(loc, expected_value, desired_value);
}
};
// Interpret the bit pattern of input (type U) as type V. Requires the size
// of V >= size of U (compile-time checked).
// Reproduced here from utils.h to keep dependencies small.
template<typename U, typename V>
static inline V bit_cast_atomic(U in) {
COMPILE_ASSERT(sizeof(U) == sizeof(V), size_of_u_not_eq_size_of_v);
union {
U u;
V v;
} tmp;
tmp.u = in;
return tmp.v;
}
template<class T> struct AtomicHelper<8, T> {
friend class Atomic<T>;
private:
COMPILE_ASSERT(sizeof(T) == 8, bad_large_atomic_helper_arg);
static T LoadRelaxed(const volatile T* loc) {
// sizeof(T) == 8
volatile const int64_t* loc_ptr =
reinterpret_cast<volatile const int64_t*>(loc);
return bit_cast_atomic<int64_t, T>(QuasiAtomic::Read64(loc_ptr));
}
static void StoreRelaxed(volatile T* loc, T desired) {
// sizeof(T) == 8
volatile int64_t* loc_ptr =
reinterpret_cast<volatile int64_t*>(loc);
QuasiAtomic::Write64(loc_ptr, bit_cast_atomic<T, int64_t>(desired));
}
static bool CompareExchangeStrongSequentiallyConsistent(volatile T* loc,
T expected_value, T desired_value) {
// sizeof(T) == 8
volatile int64_t* loc_ptr = reinterpret_cast<volatile int64_t*>(loc);
return QuasiAtomic::Cas64(bit_cast_atomic<T, int64_t>(expected_value),
bit_cast_atomic<T, int64_t>(desired_value),
loc_ptr);
}
};
template<typename T>
class PACKED(sizeof(T)) Atomic {
private:
COMPILE_ASSERT(sizeof(T) <= 4 || sizeof(T) == 8, bad_atomic_arg);
public:
Atomic<T>() : value_(0) { }
explicit Atomic<T>(T value) : value_(value) { }
// Load from memory without ordering or synchronization constraints.
T LoadRelaxed() const {
return AtomicHelper<sizeof(T), T>::LoadRelaxed(&value_);
}
// Word tearing allowed, but may race.
T LoadJavaData() const {
return value_;
}
// Load from memory with a total ordering.
T LoadSequentiallyConsistent() const;
// Store to memory without ordering or synchronization constraints.
void StoreRelaxed(T desired) {
AtomicHelper<sizeof(T), T>::StoreRelaxed(&value_, desired);
}
// Word tearing allowed, but may race.
void StoreJavaData(T desired) {
value_ = desired;
}
// Store to memory with release ordering.
void StoreRelease(T desired);
// Store to memory with a total ordering.
void StoreSequentiallyConsistent(T desired);
// Atomically replace the value with desired value if it matches the expected value.
// Participates in total ordering of atomic operations.
bool CompareExchangeStrongSequentiallyConsistent(T expected_value, T desired_value) {
return AtomicHelper<sizeof(T), T>::
CompareExchangeStrongSequentiallyConsistent(&value_, expected_value, desired_value);
}
// The same, but may fail spuriously.
bool CompareExchangeWeakSequentiallyConsistent(T expected_value, T desired_value) {
// TODO: Take advantage of the fact that it may fail spuriously.
return AtomicHelper<sizeof(T), T>::
CompareExchangeStrongSequentiallyConsistent(&value_, expected_value, desired_value);
}
// Atomically replace the value with desired value if it matches the expected value. Doesn't
// imply ordering or synchronization constraints.
bool CompareExchangeStrongRelaxed(T expected_value, T desired_value) {
// TODO: make this relaxed.
return CompareExchangeStrongSequentiallyConsistent(expected_value, desired_value);
}
// The same, but may fail spuriously.
bool CompareExchangeWeakRelaxed(T expected_value, T desired_value) {
// TODO: Take advantage of the fact that it may fail spuriously.
// TODO: make this relaxed.
return CompareExchangeStrongSequentiallyConsistent(expected_value, desired_value);
}
// Atomically replace the value with desired value if it matches the expected value. Prior accesses
// made to other memory locations by the thread that did the release become visible in this
// thread.
bool CompareExchangeWeakAcquire(T expected_value, T desired_value) {
// TODO: make this acquire.
return CompareExchangeWeakSequentiallyConsistent(expected_value, desired_value);
}
// Atomically replace the value with desired value if it matches the expected value. Prior accesses
// to other memory locations become visible to the threads that do a consume or an acquire on the
// same location.
bool CompareExchangeWeakRelease(T expected_value, T desired_value) {
// TODO: make this release.
return CompareExchangeWeakSequentiallyConsistent(expected_value, desired_value);
}
volatile T* Address() {
return &value_;
}
T FetchAndAddSequentiallyConsistent(const T value) {
if (sizeof(T) <= 4) {
return __sync_fetch_and_add(&value_, value); // Return old value.
} else {
T expected;
do {
expected = LoadRelaxed();
} while (!CompareExchangeWeakSequentiallyConsistent(expected, expected + value));
return expected;
}
}
T FetchAndSubSequentiallyConsistent(const T value) {
if (sizeof(T) <= 4) {
return __sync_fetch_and_sub(&value_, value); // Return old value.
} else {
return FetchAndAddSequentiallyConsistent(-value);
}
}
T FetchAndOrSequentiallyConsistent(const T value) {
if (sizeof(T) <= 4) {
return __sync_fetch_and_or(&value_, value); // Return old value.
} else {
T expected;
do {
expected = LoadRelaxed();
} while (!CompareExchangeWeakSequentiallyConsistent(expected, expected | value));
return expected;
}
}
T FetchAndAndSequentiallyConsistent(const T value) {
if (sizeof(T) <= 4) {
return __sync_fetch_and_and(&value_, value); // Return old value.
} else {
T expected;
do {
expected = LoadRelaxed();
} while (!CompareExchangeWeakSequentiallyConsistent(expected, expected & value));
return expected;
}
}
T operator++() { // Prefix operator.
if (sizeof(T) <= 4) {
return __sync_add_and_fetch(&value_, 1); // Return new value.
} else {
return FetchAndAddSequentiallyConsistent(1) + 1;
}
}
T operator++(int) { // Postfix operator.
return FetchAndAddSequentiallyConsistent(1);
}
T operator--() { // Prefix operator.
if (sizeof(T) <= 4) {
return __sync_sub_and_fetch(&value_, 1); // Return new value.
} else {
return FetchAndSubSequentiallyConsistent(1) - 1;
}
}
T operator--(int) { // Postfix operator.
return FetchAndSubSequentiallyConsistent(1);
}
static T MaxValue() {
return std::numeric_limits<T>::max();
}
private:
volatile T value_;
};
#endif
typedef Atomic<int32_t> AtomicInteger;
COMPILE_ASSERT(sizeof(AtomicInteger) == sizeof(int32_t), weird_atomic_int_size);
COMPILE_ASSERT(alignof(AtomicInteger) == alignof(int32_t),
atomic_int_alignment_differs_from_that_of_underlying_type);
COMPILE_ASSERT(sizeof(Atomic<int64_t>) == sizeof(int64_t), weird_atomic_int64_size);
#if defined(__LP64__)
COMPILE_ASSERT(alignof(Atomic<int64_t>) == alignof(int64_t),
atomic_int64_alignment_differs_from_that_of_underlying_type);
#endif
// The above fails on x86-32.
// This is OK, since we explicitly arrange for alignment of 8-byte fields.
#if !ART_HAVE_STDATOMIC
template<typename T>
inline T Atomic<T>::LoadSequentiallyConsistent() const {
T result = value_;
if (sizeof(T) != 8 || !QuasiAtomic::LongAtomicsUseMutexes()) {
QuasiAtomic::ThreadFenceAcquire();
// We optimistically assume this suffices for store atomicity.
// On ARMv8 we strengthen ThreadFenceAcquire to make that true.
}
return result;
}
template<typename T>
inline void Atomic<T>::StoreRelease(T desired) {
if (sizeof(T) != 8 || !QuasiAtomic::LongAtomicsUseMutexes()) {
QuasiAtomic::ThreadFenceRelease();
}
StoreRelaxed(desired);
}
template<typename T>
inline void Atomic<T>::StoreSequentiallyConsistent(T desired) {
if (sizeof(T) != 8 || !QuasiAtomic::LongAtomicsUseMutexes()) {
QuasiAtomic::ThreadFenceRelease();
}
StoreRelaxed(desired);
if (sizeof(T) != 8 || !QuasiAtomic::LongAtomicsUseMutexes()) {
QuasiAtomic::ThreadFenceSequentiallyConsistent();
}
}
#endif
} // namespace art
#endif // ART_RUNTIME_ATOMIC_H_