//===-- tsan_interface_atomic.cpp -----------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file is a part of ThreadSanitizer (TSan), a race detector. // //===----------------------------------------------------------------------===// // ThreadSanitizer atomic operations are based on C++11/C1x standards. // For background see C++11 standard. A slightly older, publicly // available draft of the standard (not entirely up-to-date, but close enough // for casual browsing) is available here: // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf // The following page contains more background information: // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_stacktrace.h" #include "sanitizer_common/sanitizer_mutex.h" #include "tsan_flags.h" #include "tsan_interface.h" #include "tsan_rtl.h" usingnamespace__tsan; #if !SANITIZER_GO && __TSAN_HAS_INT128 // Protects emulation of 128-bit atomic operations. static StaticSpinMutex mutex128; #endif #if SANITIZER_DEBUG static bool IsLoadOrder(morder mo) { return mo == mo_relaxed || mo == mo_consume || mo == mo_acquire || mo == mo_seq_cst; } static bool IsStoreOrder(morder mo) { return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst; } #endif static bool IsReleaseOrder(morder mo) { … } static bool IsAcquireOrder(morder mo) { … } static bool IsAcqRelOrder(morder mo) { … } template<typename T> T func_xchg(volatile T *v, T op) { … } template<typename T> T func_add(volatile T *v, T op) { … } template<typename T> T func_sub(volatile T *v, T op) { … } template<typename T> T func_and(volatile T *v, T op) { … } template<typename T> T func_or(volatile T *v, T op) { … } template<typename T> T func_xor(volatile T *v, T op) { … } template<typename T> T func_nand(volatile T *v, T op) { … } template<typename T> T func_cas(volatile T *v, T cmp, T xch) { … } // clang does not support 128-bit atomic ops. // Atomic ops are executed under tsan internal mutex, // here we assume that the atomic variables are not accessed // from non-instrumented code. #if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO \ && __TSAN_HAS_INT128 a128 func_xchg(volatile a128 *v, a128 op) { … } a128 func_add(volatile a128 *v, a128 op) { … } a128 func_sub(volatile a128 *v, a128 op) { … } a128 func_and(volatile a128 *v, a128 op) { … } a128 func_or(volatile a128 *v, a128 op) { … } a128 func_xor(volatile a128 *v, a128 op) { … } a128 func_nand(volatile a128 *v, a128 op) { … } a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) { … } #endif template <typename T> static int AccessSize() { … } #if !SANITIZER_GO static atomic_uint8_t *to_atomic(const volatile a8 *a) { … } static atomic_uint16_t *to_atomic(const volatile a16 *a) { … } #endif static atomic_uint32_t *to_atomic(const volatile a32 *a) { … } static atomic_uint64_t *to_atomic(const volatile a64 *a) { … } static memory_order to_mo(morder mo) { … } template<typename T> static T NoTsanAtomicLoad(const volatile T *a, morder mo) { … } #if __TSAN_HAS_INT128 && !SANITIZER_GO static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) { … } #endif template <typename T> static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) { … } template<typename T> static void NoTsanAtomicStore(volatile T *a, T v, morder mo) { … } #if __TSAN_HAS_INT128 && !SANITIZER_GO static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) { … } #endif template <typename T> static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { … } template <typename T, T (*F)(volatile T *v, T op)> static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { … } template<typename T> static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) { … } template<typename T> static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) { … } template<typename T> static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) { … } template<typename T> static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) { … } template<typename T> static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) { … } template<typename T> static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) { … } template<typename T> static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) { … } template<typename T> static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { … } template<typename T> static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { … } template<typename T> static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { … } template<typename T> static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { … } template<typename T> static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { … } template<typename T> static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { … } template<typename T> static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { … } template<typename T> static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) { … } #if __TSAN_HAS_INT128 static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v, morder mo, morder fmo) { … } #endif template<typename T> static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) { … } template <typename T> static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v, morder mo, morder fmo) { … } template<typename T> static T AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T c, T v, morder mo, morder fmo) { … } #if !SANITIZER_GO static void NoTsanAtomicFence(morder mo) { … } static void AtomicFence(ThreadState *thr, uptr pc, morder mo) { … } #endif // Interface functions follow. #if !SANITIZER_GO // C/C++ static morder convert_morder(morder mo) { … } #define ATOMIC_IMPL(func, ...) … extern "C" { SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) { … } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) { … } #endif SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) { … } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) { … } #endif SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) { … } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) { … } #endif SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) { … } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) { … } #endif SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) { … } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) { … } #endif SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) { … } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) { … } #endif SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) { … } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) { … } #endif SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) { … } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) { … } #endif SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) { … } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) { … } #endif SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, morder mo, morder fmo) { … } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v, morder mo, morder fmo) { … } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v, morder mo, morder fmo) { … } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v, morder mo, morder fmo) { … } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v, morder mo, morder fmo) { … } #endif SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, morder mo, morder fmo) { … } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v, morder mo, morder fmo) { … } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v, morder mo, morder fmo) { … } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v, morder mo, morder fmo) { … } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v, morder mo, morder fmo) { … } #endif SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, morder mo, morder fmo) { … } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, morder mo, morder fmo) { … } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, morder mo, morder fmo) { … } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, morder mo, morder fmo) { … } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v, morder mo, morder fmo) { … } #endif SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic_thread_fence(morder mo) { … } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic_signal_fence(morder mo) { … } } // extern "C" #else // #if !SANITIZER_GO // Go #define ATOMIC … #define ATOMIC_RET … extern "C" { SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { ATOMIC_RET(Load, *(a32*)(a+8), *(a32**)a, mo_acquire); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { ATOMIC_RET(Load, *(a64*)(a+8), *(a64**)a, mo_acquire); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { ATOMIC(Store, *(a32**)a, *(a32*)(a+8), mo_release); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { ATOMIC(Store, *(a64**)a, *(a64*)(a+8), mo_release); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { ATOMIC_RET(FetchAdd, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { ATOMIC_RET(FetchAnd, *(a32 *)(a + 16), *(a32 **)a, *(a32 *)(a + 8), mo_acq_rel); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { ATOMIC_RET(FetchAnd, *(a64 *)(a + 16), *(a64 **)a, *(a64 *)(a + 8), mo_acq_rel); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { ATOMIC_RET(FetchOr, *(a32 *)(a + 16), *(a32 **)a, *(a32 *)(a + 8), mo_acq_rel); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { ATOMIC_RET(FetchOr, *(a64 *)(a + 16), *(a64 **)a, *(a64 *)(a + 8), mo_acq_rel); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { ATOMIC_RET(Exchange, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_compare_exchange( ThreadState *thr, uptr cpc, uptr pc, u8 *a) { a32 cur = 0; a32 cmp = *(a32*)(a+8); ATOMIC_RET(CAS, cur, *(a32**)a, cmp, *(a32*)(a+12), mo_acq_rel, mo_acquire); *(bool*)(a+16) = (cur == cmp); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_compare_exchange( ThreadState *thr, uptr cpc, uptr pc, u8 *a) { a64 cur = 0; a64 cmp = *(a64*)(a+8); ATOMIC_RET(CAS, cur, *(a64**)a, cmp, *(a64*)(a+16), mo_acq_rel, mo_acquire); *(bool*)(a+24) = (cur == cmp); } } // extern "C" #endif // #if !SANITIZER_GO
Codebase Browser
llvm