tsan/asan: unify atomics (move atomics from tsan to sanitizer_common)
git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@159437 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/asan/asan_allocator.cc b/lib/asan/asan_allocator.cc
index 25d98bb..acd466e 100644
--- a/lib/asan/asan_allocator.cc
+++ b/lib/asan/asan_allocator.cc
@@ -34,6 +34,7 @@
#include "asan_stats.h"
#include "asan_thread.h"
#include "asan_thread_registry.h"
+#include "sanitizer_common/sanitizer_atomic.h"
#if defined(_WIN32) && !defined(__clang__)
#include <intrin.h>
@@ -159,8 +160,8 @@
struct ChunkBase {
// First 8 bytes.
uptr chunk_state : 8;
- uptr size_class : 8;
uptr alloc_tid : 24;
+ uptr size_class : 8;
uptr free_tid : 24;
// Second 8 bytes.
@@ -420,7 +421,7 @@
private:
PageGroup *FindPageGroupUnlocked(uptr addr) {
- int n = n_page_groups_;
+ int n = atomic_load(&n_page_groups_, memory_order_relaxed);
// If the page groups are not sorted yet, sort them.
if (n_sorted_page_groups_ < n) {
SortArray((uptr*)page_groups_, n);
@@ -562,9 +563,9 @@
pg->end = pg->beg + mmap_size;
pg->size_of_chunk = size;
pg->last_chunk = (uptr)(mem + size * (n_chunks - 1));
- int page_group_idx = AtomicInc(&n_page_groups_) - 1;
- CHECK(page_group_idx < (int)ASAN_ARRAY_SIZE(page_groups_));
- page_groups_[page_group_idx] = pg;
+ int idx = atomic_fetch_add(&n_page_groups_, 1, memory_order_relaxed);
+ CHECK(idx < (int)ASAN_ARRAY_SIZE(page_groups_));
+ page_groups_[idx] = pg;
return res;
}
@@ -573,7 +574,7 @@
AsanLock mu_;
PageGroup *page_groups_[kMaxAvailableRam / kMinMmapSize];
- int n_page_groups_; // atomic
+ atomic_uint32_t n_page_groups_;
int n_sorted_page_groups_;
};
@@ -721,7 +722,8 @@
AsanChunk *m = PtrToChunk((uptr)ptr);
// Flip the chunk_state atomically to avoid race on double-free.
- u8 old_chunk_state = AtomicExchange((u8*)m, CHUNK_QUARANTINE);
+ u8 old_chunk_state = atomic_exchange((atomic_uint8_t*)m, CHUNK_QUARANTINE,
+ memory_order_acq_rel);
if (old_chunk_state == CHUNK_QUARANTINE) {
AsanReport("ERROR: AddressSanitizer attempting double-free on %p:\n", ptr);
diff --git a/lib/asan/asan_rtl.cc b/lib/asan/asan_rtl.cc
index 71e05a7..616266c 100644
--- a/lib/asan/asan_rtl.cc
+++ b/lib/asan/asan_rtl.cc
@@ -21,14 +21,15 @@
#include "asan_stats.h"
#include "asan_thread.h"
#include "asan_thread_registry.h"
+#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_libc.h"
namespace __sanitizer {
using namespace __asan;
void Die() {
- static int num_calls = 0;
- if (AtomicInc(&num_calls) > 1) {
+ static atomic_uint32_t num_calls;
+ if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) {
// Don't die twice - run a busy loop.
while (1) { }
}
@@ -343,8 +344,8 @@
void __asan_report_error(uptr pc, uptr bp, uptr sp,
uptr addr, bool is_write, uptr access_size) {
// Do not print more than one report, otherwise they will mix up.
- static int num_calls = 0;
- if (AtomicInc(&num_calls) > 1) return;
+ static atomic_uint32_t num_calls;
+ if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) return;
AsanPrintf("===================================================="
"=============\n");
diff --git a/lib/tsan/rtl/tsan_atomic.h b/lib/sanitizer_common/sanitizer_atomic.h
similarity index 72%
rename from lib/tsan/rtl/tsan_atomic.h
rename to lib/sanitizer_common/sanitizer_atomic.h
index 6fcd9f9..df3344e 100644
--- a/lib/tsan/rtl/tsan_atomic.h
+++ b/lib/sanitizer_common/sanitizer_atomic.h
@@ -1,4 +1,4 @@
-//===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
+//===-- sanitizer_atomic.h --------------------------------------*- C++ -*-===//
//
// The LLVM Compiler Infrastructure
//
@@ -6,20 +6,13 @@
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
-//
-// This file is a part of ThreadSanitizer (TSan), a race detector.
-//
-// Atomic operations. For now implies IA-32/Intel64.
-//===----------------------------------------------------------------------===//
-#ifndef TSAN_ATOMIC_H
-#define TSAN_ATOMIC_H
+#ifndef SANITIZER_ATOMIC_H
+#define SANITIZER_ATOMIC_H
-#include "tsan_defs.h"
+#include "sanitizer_internal_defs.h"
-namespace __tsan {
-
-const int kCacheLineSize = 64;
+namespace __sanitizer {
enum memory_order {
memory_order_relaxed = 1 << 0,
@@ -30,6 +23,16 @@
memory_order_seq_cst = 1 << 5,
};
+struct atomic_uint8_t {
+ typedef u8 Type;
+ volatile Type val_dont_use;
+};
+
+struct atomic_uint16_t {
+ typedef u16 Type;
+ volatile Type val_dont_use;
+};
+
struct atomic_uint32_t {
typedef u32 Type;
volatile Type val_dont_use;
@@ -50,13 +53,15 @@
}
INLINE void atomic_thread_fence(memory_order) {
- __asm__ __volatile__("mfence" ::: "memory");
+ __sync_synchronize();
}
INLINE void proc_yield(int cnt) {
__asm__ __volatile__("" ::: "memory");
+#if defined(__i386__) || defined(__x86_64__)
for (int i = 0; i < cnt; i++)
__asm__ __volatile__("pause");
+#endif
__asm__ __volatile__("" ::: "memory");
}
@@ -109,9 +114,15 @@
return __sync_fetch_and_add(&a->val_dont_use, -v);
}
-INLINE uptr atomic_exchange(volatile atomic_uintptr_t *a, uptr v,
- memory_order mo) {
- __asm__ __volatile__("xchg %1, %0" : "+r"(v), "+m"(*a) : : "memory", "cc");
+template<typename T>
+INLINE typename T::Type atomic_exchange(volatile T *a,
+ typename T::Type v, memory_order mo) {
+ DCHECK(!((uptr)a % sizeof(*a)));
+ if (mo & (memory_order_release | memory_order_acq_rel | memory_order_seq_cst))
+ __sync_synchronize();
+ v = __sync_lock_test_and_set(&a->val_dont_use, v);
+ if (mo == memory_order_seq_cst)
+ __sync_synchronize();
return v;
}
@@ -129,12 +140,14 @@
return false;
}
-INLINE bool atomic_compare_exchange_weak(volatile atomic_uintptr_t *a,
- uptr *cmp, uptr xchg,
- memory_order mo) {
+template<typename T>
+INLINE bool atomic_compare_exchange_weak(volatile T *a,
+ typename T::Type *cmp,
+ typename T::Type xchg,
+ memory_order mo) {
return atomic_compare_exchange_strong(a, cmp, xchg, mo);
}
-} // namespace __tsan
+} // namespace __sanitizer
-#endif // TSAN_ATOMIC_H
+#endif // SANITIZER_ATOMIC_H
diff --git a/lib/sanitizer_common/sanitizer_common.h b/lib/sanitizer_common/sanitizer_common.h
index f24539b..e691f48 100644
--- a/lib/sanitizer_common/sanitizer_common.h
+++ b/lib/sanitizer_common/sanitizer_common.h
@@ -77,11 +77,6 @@
int Atexit(void (*function)(void));
void SortArray(uptr *array, uptr size);
-// Atomics
-int AtomicInc(int *a);
-u16 AtomicExchange(u16 *a, u16 new_val);
-u8 AtomicExchange(u8 *a, u8 new_val);
-
// Math
inline bool IsPowerOfTwo(uptr x) {
return (x & (x - 1)) == 0;
diff --git a/lib/sanitizer_common/sanitizer_internal_defs.h b/lib/sanitizer_common/sanitizer_internal_defs.h
index ea914ab..b8cf61f 100644
--- a/lib/sanitizer_common/sanitizer_internal_defs.h
+++ b/lib/sanitizer_common/sanitizer_internal_defs.h
@@ -112,6 +112,24 @@
#define CHECK_GT(a, b) CHECK_IMPL((a), >, (b))
#define CHECK_GE(a, b) CHECK_IMPL((a), >=, (b))
+#if TSAN_DEBUG
+#define DCHECK(a) CHECK(a)
+#define DCHECK_EQ(a, b) CHECK_EQ(a, b)
+#define DCHECK_NE(a, b) CHECK_NE(a, b)
+#define DCHECK_LT(a, b) CHECK_LT(a, b)
+#define DCHECK_LE(a, b) CHECK_LE(a, b)
+#define DCHECK_GT(a, b) CHECK_GT(a, b)
+#define DCHECK_GE(a, b) CHECK_GE(a, b)
+#else
+#define DCHECK(a)
+#define DCHECK_EQ(a, b)
+#define DCHECK_NE(a, b)
+#define DCHECK_LT(a, b)
+#define DCHECK_LE(a, b)
+#define DCHECK_GT(a, b)
+#define DCHECK_GE(a, b)
+#endif
+
#define UNIMPLEMENTED() CHECK("unimplemented" && 0)
#define COMPILER_CHECK(pred) IMPL_COMPILER_ASSERT(pred, __LINE__)
diff --git a/lib/tsan/rtl/tsan_defs.h b/lib/tsan/rtl/tsan_defs.h
index c245031..3d9cd54 100644
--- a/lib/tsan/rtl/tsan_defs.h
+++ b/lib/tsan/rtl/tsan_defs.h
@@ -54,24 +54,6 @@
const bool kCollectStats = false;
#endif
-#if TSAN_DEBUG
-#define DCHECK(a) CHECK(a)
-#define DCHECK_EQ(a, b) CHECK_EQ(a, b)
-#define DCHECK_NE(a, b) CHECK_NE(a, b)
-#define DCHECK_LT(a, b) CHECK_LT(a, b)
-#define DCHECK_LE(a, b) CHECK_LE(a, b)
-#define DCHECK_GT(a, b) CHECK_GT(a, b)
-#define DCHECK_GE(a, b) CHECK_GE(a, b)
-#else
-#define DCHECK(a)
-#define DCHECK_EQ(a, b)
-#define DCHECK_NE(a, b)
-#define DCHECK_LT(a, b)
-#define DCHECK_LE(a, b)
-#define DCHECK_GT(a, b)
-#define DCHECK_GE(a, b)
-#endif
-
// The following "build consistency" machinery ensures that all source files
// are built in the same configuration. Inconsistent builds lead to
// hard to debug crashes.
diff --git a/lib/tsan/rtl/tsan_interceptors.cc b/lib/tsan/rtl/tsan_interceptors.cc
index 2e9426b..5dbaf86 100644
--- a/lib/tsan/rtl/tsan_interceptors.cc
+++ b/lib/tsan/rtl/tsan_interceptors.cc
@@ -12,11 +12,11 @@
//===----------------------------------------------------------------------===//
#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_placement_new.h"
#include "tsan_rtl.h"
#include "tsan_interface.h"
-#include "tsan_atomic.h"
#include "tsan_platform.h"
#include "tsan_mman.h"
diff --git a/lib/tsan/rtl/tsan_mutex.h b/lib/tsan/rtl/tsan_mutex.h
index 2180978..6fbf822 100644
--- a/lib/tsan/rtl/tsan_mutex.h
+++ b/lib/tsan/rtl/tsan_mutex.h
@@ -13,7 +13,7 @@
#ifndef TSAN_MUTEX_H
#define TSAN_MUTEX_H
-#include "tsan_atomic.h"
+#include "sanitizer_common/sanitizer_atomic.h"
#include "tsan_defs.h"
namespace __tsan {
diff --git a/lib/tsan/rtl/tsan_rtl.cc b/lib/tsan/rtl/tsan_rtl.cc
index a66f672..822dfa1 100644
--- a/lib/tsan/rtl/tsan_rtl.cc
+++ b/lib/tsan/rtl/tsan_rtl.cc
@@ -12,6 +12,7 @@
// Main file (entry points) for the TSan run-time.
//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_placement_new.h"
@@ -19,7 +20,6 @@
#include "tsan_platform.h"
#include "tsan_rtl.h"
#include "tsan_interface.h"
-#include "tsan_atomic.h"
#include "tsan_mman.h"
#include "tsan_suppressions.h"
diff --git a/lib/tsan/rtl/tsan_sync.h b/lib/tsan/rtl/tsan_sync.h
index 51989d2..34d3e0b 100644
--- a/lib/tsan/rtl/tsan_sync.h
+++ b/lib/tsan/rtl/tsan_sync.h
@@ -13,7 +13,8 @@
#ifndef TSAN_SYNC_H
#define TSAN_SYNC_H
-#include "tsan_atomic.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
#include "tsan_clock.h"
#include "tsan_defs.h"
#include "tsan_mutex.h"
diff --git a/lib/tsan/rtl_tests/tsan_mutex.cc b/lib/tsan/rtl_tests/tsan_mutex.cc
index 8fa4670..4d9c779 100644
--- a/lib/tsan/rtl_tests/tsan_mutex.cc
+++ b/lib/tsan/rtl_tests/tsan_mutex.cc
@@ -10,7 +10,7 @@
// This file is a part of ThreadSanitizer (TSan), a race detector.
//
//===----------------------------------------------------------------------===//
-#include "tsan_atomic.h"
+#include "sanitizer_common/sanitizer_atomic.h"
#include "tsan_interface.h"
#include "tsan_interface_ann.h"
#include "tsan_test_util.h"
diff --git a/lib/tsan/rtl_tests/tsan_test_util_linux.cc b/lib/tsan/rtl_tests/tsan_test_util_linux.cc
index 35a7c48..5bc393b 100644
--- a/lib/tsan/rtl_tests/tsan_test_util_linux.cc
+++ b/lib/tsan/rtl_tests/tsan_test_util_linux.cc
@@ -1,3 +1,4 @@
+
//===-- tsan_test_util_linux.cc -------------------------------------------===//
//
// The LLVM Compiler Infrastructure
@@ -12,9 +13,9 @@
// Test utils, linux implementation.
//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_atomic.h"
#include "tsan_interface.h"
#include "tsan_test_util.h"
-#include "tsan_atomic.h"
#include "tsan_report.h"
#include "gtest/gtest.h"
diff --git a/lib/tsan/unit_tests/tsan_mutex_test.cc b/lib/tsan/unit_tests/tsan_mutex_test.cc
index 1e11ee8..e26dd48 100644
--- a/lib/tsan/unit_tests/tsan_mutex_test.cc
+++ b/lib/tsan/unit_tests/tsan_mutex_test.cc
@@ -11,7 +11,8 @@
//
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_internal_defs.h"
-#include "tsan_atomic.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
#include "tsan_mutex.h"
#include "gtest/gtest.h"
