//===-- tsan_interceptors_posix.cpp ---------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file is a part of ThreadSanitizer (TSan), a race detector. // // FIXME: move as many interceptors as possible into // sanitizer_common/sanitizer_common_interceptors.inc //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_allocator_dlsym.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_glibc_version.h" #include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_platform_limits_netbsd.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" #include "sanitizer_common/sanitizer_posix.h" #include "sanitizer_common/sanitizer_stacktrace.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" #include "interception/interception.h" #include "tsan_interceptors.h" #include "tsan_interface.h" #include "tsan_platform.h" #include "tsan_suppressions.h" #include "tsan_rtl.h" #include "tsan_mman.h" #include "tsan_fd.h" #include <stdarg.h> usingnamespace__tsan; DECLARE_REAL(…) DECLARE_REAL(…) #if SANITIZER_FREEBSD || SANITIZER_APPLE #define stdout … #define stderr … #endif #if SANITIZER_NETBSD #define dirfd … #define fileno_unlocked … #define stdout … #define stderr … #define nanosleep … #define vfork … #endif #ifdef __mips__ const int kSigCount = 129; #else const int kSigCount = …; #endif #ifdef __mips__ struct ucontext_t { u64 opaque[768 / sizeof(u64) + 1]; }; #else struct ucontext_t { … }; #endif #if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 || \ defined(__s390x__) #define PTHREAD_ABI_BASE … #elif defined(__aarch64__) || SANITIZER_PPC64V2 #define PTHREAD_ABI_BASE … #elif SANITIZER_LOONGARCH64 #define PTHREAD_ABI_BASE … #elif SANITIZER_RISCV64 #define PTHREAD_ABI_BASE … #endif extern "C" int pthread_attr_init(void *attr); extern "C" int pthread_attr_destroy(void *attr); DECLARE_REAL(…) extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize); extern "C" int pthread_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(void)); extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v)); extern "C" int pthread_setspecific(unsigned key, const void *v); DECLARE_REAL(…) DECLARE_REAL(…) DECLARE_REAL_AND_INTERCEPTOR(…) DECLARE_REAL_AND_INTERCEPTOR(…) extern "C" int pthread_equal(void *t1, void *t2); extern "C" void *pthread_self(); extern "C" void _exit(int status); #if !SANITIZER_NETBSD extern "C" int fileno_unlocked(void *stream); extern "C" int dirfd(void *dirp); #endif #if SANITIZER_NETBSD extern __sanitizer_FILE __sF[]; #else extern __sanitizer_FILE *stdout, *stderr; #endif #if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD const int PTHREAD_MUTEX_RECURSIVE = …; const int PTHREAD_MUTEX_RECURSIVE_NP = …; #else const int PTHREAD_MUTEX_RECURSIVE = 2; const int PTHREAD_MUTEX_RECURSIVE_NP = 2; #endif #if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD const int EPOLL_CTL_ADD = …; #endif const int SIGILL = …; const int SIGTRAP = …; const int SIGABRT = …; const int SIGFPE = …; const int SIGSEGV = …; const int SIGPIPE = …; const int SIGTERM = …; #if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD const int SIGBUS = 10; const int SIGSYS = 12; #else const int SIGBUS = …; const int SIGSYS = …; #endif #if SANITIZER_HAS_SIGINFO const int SI_TIMER = …; #endif void *const MAP_FAILED = …; #if SANITIZER_NETBSD const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567; #elif !SANITIZER_APPLE const int PTHREAD_BARRIER_SERIAL_THREAD = …; #endif const int MAP_FIXED = …; long_t; mode_t; // From /usr/include/unistd.h #define F_ULOCK … #define F_LOCK … #define F_TLOCK … #define F_TEST … #if SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD const int SA_SIGINFO = 0x40; const int SIG_SETMASK = 3; #elif defined(__mips__) const int SA_SIGINFO = 8; const int SIG_SETMASK = 3; #else const int SA_SIGINFO = …; const int SIG_SETMASK = …; #endif namespace __tsan { struct SignalDesc { … }; struct ThreadSignalContext { … }; void EnterBlockingFunc(ThreadState *thr) { … } // The sole reason tsan wraps atexit callbacks is to establish synchronization // between callback setup and callback execution. struct AtExitCtx { … }; // InterceptorContext holds all global data required for interceptors. // It's explicitly constructed in InitializeInterceptors with placement new // and is never destroyed. This allows usage of members with non-trivial // constructors and destructors. struct InterceptorContext { … }; alignas(64) static char interceptor_placeholder[sizeof(InterceptorContext)]; InterceptorContext *interceptor_ctx() { … } LibIgnore *libignore() { … } void InitializeLibIgnore() { … } // The following two hooks can be used by for cooperative scheduling when // locking. #ifdef TSAN_EXTERNAL_HOOKS void OnPotentiallyBlockingRegionBegin(); void OnPotentiallyBlockingRegionEnd(); #else SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionBegin() { … } SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() { … } #endif // FIXME: Use for `in_symbolizer()` as well. As-is we can't use // `DlSymAllocator`, because it uses the primary allocator only. Symbolizer // requires support of the secondary allocator for larger blocks. struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> { … }; } // namespace __tsan static ThreadSignalContext *SigCtx(ThreadState *thr) { … } ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc) : … { … } ScopedInterceptor::~ScopedInterceptor() { … } NOINLINE void ScopedInterceptor::EnableIgnoresImpl() { … } NOINLINE void ScopedInterceptor::DisableIgnoresImpl() { … } #define TSAN_INTERCEPT(func) … #if SANITIZER_FREEBSD || SANITIZER_NETBSD #define TSAN_INTERCEPT_VER … #else #define TSAN_INTERCEPT_VER(func, ver) … #endif #if SANITIZER_FREEBSD #define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS … #else #define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func) … #endif #if SANITIZER_NETBSD #define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS … #define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR … #else #define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) … #define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) … #endif #define READ_STRING_OF_LEN(thr, pc, s, len, n) … #define READ_STRING(thr, pc, s, n) … #define BLOCK_REAL(name) … struct BlockingCall { … }; TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) { … } TSAN_INTERCEPTOR(int, usleep, long_t usec) { … } TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) { … } TSAN_INTERCEPTOR(int, pause, int fake) { … } // Note: we specifically call the function in such strange way // with "installed_at" because in reports it will appear between // callback frames and the frame that installed the callback. static void at_exit_callback_installed_at() { … } static void cxa_at_exit_callback_installed_at(void *arg) { … } static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), void *arg, void *dso); #if !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, atexit, void (*f)()) { … } #endif TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { … } static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), void *arg, void *dso) { … } #if !SANITIZER_APPLE && !SANITIZER_NETBSD static void on_exit_callback_installed_at(int status, void *arg) { … } TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { … } #define TSAN_MAYBE_INTERCEPT_ON_EXIT … #else #define TSAN_MAYBE_INTERCEPT_ON_EXIT #endif // Cleanup old bufs. static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) { … } static void SetJmp(ThreadState *thr, uptr sp) { … } static void LongJmp(ThreadState *thr, uptr *env) { … } // FIXME: put everything below into a common extern "C" block? extern "C" void __tsan_setjmp(uptr sp) { … } #if SANITIZER_APPLE TSAN_INTERCEPTOR(int, setjmp, void *env); TSAN_INTERCEPTOR(int, _setjmp, void *env); TSAN_INTERCEPTOR(int, sigsetjmp, void *env); #else // SANITIZER_APPLE #if SANITIZER_NETBSD #define setjmp_symname … #define sigsetjmp_symname … #else #define setjmp_symname … #define sigsetjmp_symname … #endif DEFINE_REAL(…) DEFINE_REAL(…) DEFINE_REAL(…) #if !SANITIZER_NETBSD DEFINE_REAL(…) #endif // The real interceptor for setjmp is special, and implemented in pure asm. We // just need to initialize the REAL functions so that they can be used in asm. static void InitializeSetjmpInterceptors() { … } #endif // SANITIZER_APPLE #if SANITIZER_NETBSD #define longjmp_symname … #define siglongjmp_symname … #else #define longjmp_symname … #define siglongjmp_symname … #endif TSAN_INTERCEPTOR(void, longjmp_symname, uptr *env, int val) { … } TSAN_INTERCEPTOR(void, siglongjmp_symname, uptr *env, int val) { … } #if SANITIZER_NETBSD TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) { { SCOPED_INTERCEPTOR_RAW(_longjmp, env, val); } LongJmp(cur_thread(), env); REAL(_longjmp)(env, val); } #endif #if !SANITIZER_APPLE TSAN_INTERCEPTOR(void*, malloc, uptr size) { … } // In glibc<2.25, dynamic TLS blocks are allocated by __libc_memalign. Intercept // __libc_memalign so that (1) we can detect races (2) free will not be called // on libc internally allocated blocks. TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { … } TSAN_INTERCEPTOR(void *, calloc, uptr n, uptr size) { … } TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { … } TSAN_INTERCEPTOR(void *, reallocarray, void *p, uptr n, uptr size) { … } TSAN_INTERCEPTOR(void, free, void *p) { … } TSAN_INTERCEPTOR(void, cfree, void *p) { … } TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) { … } #endif TSAN_INTERCEPTOR(char *, strcpy, char *dst, const char *src) { … } TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) { … } TSAN_INTERCEPTOR(char*, strdup, const char *str) { … } // Zero out addr if it points into shadow memory and was provided as a hint // only, i.e., MAP_FIXED is not set. static bool fix_mmap_addr(void **addr, long_t sz, int flags) { … } template <class Mmap> static void *mmap_interceptor(ThreadState *thr, uptr pc, Mmap real_mmap, void *addr, SIZE_T sz, int prot, int flags, int fd, OFF64_T off) { … } template <class Munmap> static int munmap_interceptor(ThreadState *thr, uptr pc, Munmap real_munmap, void *addr, SIZE_T sz) { … } #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { … } #define TSAN_MAYBE_INTERCEPT_MEMALIGN … #else #define TSAN_MAYBE_INTERCEPT_MEMALIGN #endif #if !SANITIZER_APPLE TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { … } TSAN_INTERCEPTOR(void*, valloc, uptr sz) { … } #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { … } #define TSAN_MAYBE_INTERCEPT_PVALLOC … #else #define TSAN_MAYBE_INTERCEPT_PVALLOC #endif #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { … } #endif // Both __cxa_guard_acquire and pthread_once 0-initialize // the object initially. pthread_once does not have any // other ABI requirements. __cxa_guard_acquire assumes // that any non-0 value in the first byte means that // initialization is completed. Contents of the remaining // bytes are up to us. constexpr u32 kGuardInit = …; constexpr u32 kGuardDone = …; constexpr u32 kGuardRunning = …; constexpr u32 kGuardWaiter = …; static int guard_acquire(ThreadState *thr, uptr pc, atomic_uint32_t *g, bool blocking_hooks = true) { … } static void guard_release(ThreadState *thr, uptr pc, atomic_uint32_t *g, u32 v) { … } // __cxa_guard_acquire and friends need to be intercepted in a special way - // regular interceptors will break statically-linked libstdc++. Linux // interceptors are especially defined as weak functions (so that they don't // cause link errors when user defines them as well). So they silently // auto-disable themselves when such symbol is already present in the binary. If // we link libstdc++ statically, it will bring own __cxa_guard_acquire which // will silently replace our interceptor. That's why on Linux we simply export // these interceptors with INTERFACE_ATTRIBUTE. // On OS X, we don't support statically linking, so we just use a regular // interceptor. #if SANITIZER_APPLE #define STDCXX_INTERCEPTOR … #else #define STDCXX_INTERCEPTOR(rettype, name, ...) … #endif // Used in thread-safe function static initialization. STDCXX_INTERCEPTOR(…) { … STDCXX_INTERCEPTOR(…) { … STDCXX_INTERCEPTOR(…) { … namespace __tsan { void DestroyThreadState() { … } void PlatformCleanUpThreadState(ThreadState *thr) { … } } // namespace __tsan #if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD static void thread_finalize(void *v) { … } #endif struct ThreadParam { … }; extern "C" void *__tsan_thread_start_func(void *arg) { … } TSAN_INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), void * param) { … } TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) { … } // DEFINE_INTERNAL_PTHREAD_FUNCTIONS namespace __sanitizer { int internal_pthread_create(void *th, void *attr, void *(*callback)(void *), void *param) { … } int internal_pthread_join(void *th, void **ret) { … } } // namespace __sanitizer TSAN_INTERCEPTOR(int, pthread_detach, void *th) { … } TSAN_INTERCEPTOR(void, pthread_exit, void *retval) { … } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) { … } TSAN_INTERCEPTOR(int, pthread_timedjoin_np, void *th, void **ret, const struct timespec *abstime) { … } #endif // Problem: // NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2). // pthread_cond_t has different size in the different versions. // If call new REAL functions for old pthread_cond_t, they will corrupt memory // after pthread_cond_t (old cond is smaller). // If we call old REAL functions for new pthread_cond_t, we will lose some // functionality (e.g. old functions do not support waiting against // CLOCK_REALTIME). // Proper handling would require to have 2 versions of interceptors as well. // But this is messy, in particular requires linker scripts when sanitizer // runtime is linked into a shared library. // Instead we assume we don't have dynamic libraries built against old // pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag // that allows to work with old libraries (but this mode does not support // some features, e.g. pthread_condattr_getpshared). static void *init_cond(void *c, bool force = false) { … } namespace { template <class Fn> struct CondMutexUnlockCtx { … }; template <class Fn> void CondMutexUnlockCtx<Fn>::Unlock() const { … } } // namespace INTERCEPTOR(int, pthread_cond_init, void *c, void *a) { … } template <class Fn> int cond_wait(ThreadState *thr, uptr pc, ScopedInterceptor *si, const Fn &fn, void *c, void *m) { … } INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { … } INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) { … } #if SANITIZER_LINUX INTERCEPTOR(int, pthread_cond_clockwait, void *c, void *m, __sanitizer_clockid_t clock, void *abstime) { … } #define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT … #else #define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT #endif #if SANITIZER_APPLE INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void *c, void *m, void *reltime) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait_relative_np, cond, m, reltime); return cond_wait( thr, pc, &si, [=]() { return REAL(pthread_cond_timedwait_relative_np)(cond, m, reltime); }, cond, m); } #endif INTERCEPTOR(int, pthread_cond_signal, void *c) { … } INTERCEPTOR(int, pthread_cond_broadcast, void *c) { … } INTERCEPTOR(int, pthread_cond_destroy, void *c) { … } TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) { … } TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { … } TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) { … } TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { … } #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { … } #endif TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) { … } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, pthread_mutex_clocklock, void *m, __sanitizer_clockid_t clock, void *abstime) { … } #endif #if SANITIZER_GLIBC # if !__GLIBC_PREREQ(2, 34) // glibc 2.34 applies a non-default version for the two functions. They are no // longer expected to be intercepted by programs. TSAN_INTERCEPTOR(int, __pthread_mutex_lock, void *m) { SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_lock, m); MutexPreLock(thr, pc, (uptr)m); int res = BLOCK_REAL(__pthread_mutex_lock)(m); if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); if (res == 0 || res == errno_EOWNERDEAD) MutexPostLock(thr, pc, (uptr)m); if (res == errno_EINVAL) MutexInvalidAccess(thr, pc, (uptr)m); return res; } TSAN_INTERCEPTOR(int, __pthread_mutex_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_unlock, m); MutexUnlock(thr, pc, (uptr)m); int res = REAL(__pthread_mutex_unlock)(m); if (res == errno_EINVAL) MutexInvalidAccess(thr, pc, (uptr)m); return res; } # endif #endif #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) { … } TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) { … } TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) { … } TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) { … } TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) { … } #endif TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) { … } TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) { … } TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) { … } TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { … } #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { … } #endif TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) { … } TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { … } #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { … } #endif TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { … } #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) { … } TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) { … } TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) { … } #endif TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { … } #if SANITIZER_GLIBC TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { … } TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) { … } #define TSAN_MAYBE_INTERCEPT___FXSTAT … #else #define TSAN_MAYBE_INTERCEPT___FXSTAT #endif #if !SANITIZER_GLIBC || __GLIBC_PREREQ(2, 33) TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { … } #define TSAN_MAYBE_INTERCEPT_FSTAT … #else #define TSAN_MAYBE_INTERCEPT_FSTAT #endif #if __GLIBC_PREREQ(2, 33) TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) { … } #define TSAN_MAYBE_INTERCEPT_FSTAT64 … #else #define TSAN_MAYBE_INTERCEPT_FSTAT64 #endif TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) { … } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, open64, const char *name, int oflag, ...) { … } #define TSAN_MAYBE_INTERCEPT_OPEN64 … #else #define TSAN_MAYBE_INTERCEPT_OPEN64 #endif TSAN_INTERCEPTOR(int, creat, const char *name, int mode) { … } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) { … } #define TSAN_MAYBE_INTERCEPT_CREAT64 … #else #define TSAN_MAYBE_INTERCEPT_CREAT64 #endif TSAN_INTERCEPTOR(int, dup, int oldfd) { … } TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) { … } #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) { … } #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) { … } #define TSAN_MAYBE_INTERCEPT_EVENTFD … #else #define TSAN_MAYBE_INTERCEPT_EVENTFD #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) { … } #define TSAN_MAYBE_INTERCEPT_SIGNALFD … #else #define TSAN_MAYBE_INTERCEPT_SIGNALFD #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, inotify_init, int fake) { … } #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT … #else #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, inotify_init1, int flags) { … } #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 … #else #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 #endif TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) { … } TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) { … } TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) { … } TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) { … } TSAN_INTERCEPTOR(int, listen, int fd, int backlog) { … } TSAN_INTERCEPTOR(int, close, int fd) { … } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, __close, int fd) { … } #define TSAN_MAYBE_INTERCEPT___CLOSE … #else #define TSAN_MAYBE_INTERCEPT___CLOSE #endif // glibc guts #if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) { … } #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE … #else #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE #endif TSAN_INTERCEPTOR(int, pipe, int *pipefd) { … } #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) { … } #endif TSAN_INTERCEPTOR(int, unlink, char *path) { … } TSAN_INTERCEPTOR(void*, tmpfile, int fake) { … } #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, tmpfile64, int fake) { … } #define TSAN_MAYBE_INTERCEPT_TMPFILE64 … #else #define TSAN_MAYBE_INTERCEPT_TMPFILE64 #endif static void FlushStreams() { … } TSAN_INTERCEPTOR(void, abort, int fake) { … } TSAN_INTERCEPTOR(int, rmdir, char *path) { … } TSAN_INTERCEPTOR(int, closedir, void *dirp) { … } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, epoll_create, int size) { … } TSAN_INTERCEPTOR(int, epoll_create1, int flags) { … } TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) { … } TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { … } TSAN_INTERCEPTOR(int, epoll_pwait, int epfd, void *ev, int cnt, int timeout, void *sigmask) { … } TSAN_INTERCEPTOR(int, epoll_pwait2, int epfd, void *ev, int cnt, void *timeout, void *sigmask) { … } #define TSAN_MAYBE_INTERCEPT_EPOLL … #else #define TSAN_MAYBE_INTERCEPT_EPOLL #endif // The following functions are intercepted merely to process pending signals. // If program blocks signal X, we must deliver the signal before the function // returns. Similarly, if program unblocks a signal (or returns from sigsuspend) // it's better to deliver the signal straight away. TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) { … } TSAN_INTERCEPTOR(int, sigblock, int mask) { … } TSAN_INTERCEPTOR(int, sigsetmask, int mask) { … } TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { … } namespace __tsan { static void ReportErrnoSpoiling(ThreadState *thr, uptr pc, int sig) { … } static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, int sig, __sanitizer_siginfo *info, void *uctx) { … } void ProcessPendingSignalsImpl(ThreadState *thr) { … } } // namespace __tsan static bool is_sync_signal(ThreadSignalContext *sctx, int sig, __sanitizer_siginfo *info) { … } void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) { … } TSAN_INTERCEPTOR(int, raise, int sig) { … } TSAN_INTERCEPTOR(int, kill, int pid, int sig) { … } TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) { … } TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) { … } TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, void *hints, void *rv) { … } TSAN_INTERCEPTOR(int, fork, int fake) { … } void atfork_prepare() { … } void atfork_parent() { … } void atfork_child() { … } #if !SANITIZER_IOS TSAN_INTERCEPTOR(int, vfork, int fake) { … } #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, clone, int (*fn)(void *), void *stack, int flags, void *arg, int *parent_tid, void *tls, pid_t *child_tid) { … } #endif #if !SANITIZER_APPLE && !SANITIZER_ANDROID dl_iterate_phdr_cb_t; struct dl_iterate_phdr_data { … }; static bool IsAppNotRodata(uptr addr) { … } static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size, void *data) { … } TSAN_INTERCEPTOR(int, dl_iterate_phdr, dl_iterate_phdr_cb_t cb, void *data) { … } #endif static int OnExit(ThreadState *thr) { … } #if !SANITIZER_APPLE static void HandleRecvmsg(ThreadState *thr, uptr pc, __sanitizer_msghdr *msg) { … } #endif #include "sanitizer_common/sanitizer_platform_interceptors.h" // Causes interceptor recursion (getaddrinfo() and fopen()) #undef SANITIZER_INTERCEPT_GETADDRINFO // We define our own. #if SANITIZER_INTERCEPT_TLS_GET_ADDR #define NEED_TLS_GET_ADDR #endif #undef SANITIZER_INTERCEPT_TLS_GET_ADDR #define SANITIZER_INTERCEPT_TLS_GET_OFFSET … #undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) … #define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) … #define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) … #define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) … #define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) … #define COMMON_INTERCEPTOR_DLOPEN(filename, flag) … // Ignore interceptors in OnLibraryLoaded()/Unloaded(). These hooks use code // (ListOfModules::init, MemoryMappingLayout::DumpListOfModules) that make // intercepted calls, which can cause deadlockes with ReportRace() which also // uses this code. #define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) … #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() … #define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) … #define COMMON_INTERCEPTOR_RELEASE(ctx, u) … #define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) … #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) … #define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) … #define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) … #define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) … #define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) … #define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) … #define COMMON_INTERCEPTOR_BLOCK_REAL(name) … #define COMMON_INTERCEPTOR_ON_EXIT(ctx) … #define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \ off) … #define COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz) … #if !SANITIZER_APPLE #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) … #endif #define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) … #define COMMON_INTERCEPTOR_USER_CALLBACK_START() … #define COMMON_INTERCEPTOR_USER_CALLBACK_END() … #include "sanitizer_common/sanitizer_common_interceptors.inc" static int sigaction_impl(int sig, const __sanitizer_sigaction *act, __sanitizer_sigaction *old); static __sanitizer_sighandler_ptr signal_impl(int sig, __sanitizer_sighandler_ptr h); #define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) … #define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) … #define SIGNAL_INTERCEPTOR_ENTER() … #include "sanitizer_common/sanitizer_signal_interceptors.inc" int sigaction_impl(int sig, const __sanitizer_sigaction *act, __sanitizer_sigaction *old) { … } static __sanitizer_sighandler_ptr signal_impl(int sig, __sanitizer_sighandler_ptr h) { … } #define TSAN_SYSCALL() … struct ScopedSyscall { … }; #if !SANITIZER_FREEBSD && !SANITIZER_APPLE static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { … } static USED void syscall_acquire(uptr pc, uptr addr) { … } static USED void syscall_release(uptr pc, uptr addr) { … } static void syscall_fd_close(uptr pc, int fd) { … } static USED void syscall_fd_acquire(uptr pc, int fd) { … } static USED void syscall_fd_release(uptr pc, int fd) { … } static USED void sycall_blocking_start() { … } static USED void sycall_blocking_end() { … } static void syscall_pre_fork(uptr pc) { … } static void syscall_post_fork(uptr pc, int pid) { … } #endif #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) … #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) … #define COMMON_SYSCALL_POST_READ_RANGE(p, s) … #define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) … #define COMMON_SYSCALL_ACQUIRE(addr) … #define COMMON_SYSCALL_RELEASE(addr) … #define COMMON_SYSCALL_FD_CLOSE(fd) … #define COMMON_SYSCALL_FD_ACQUIRE(fd) … #define COMMON_SYSCALL_FD_RELEASE(fd) … #define COMMON_SYSCALL_PRE_FORK() … #define COMMON_SYSCALL_POST_FORK(res) … #define COMMON_SYSCALL_BLOCKING_START() … #define COMMON_SYSCALL_BLOCKING_END() … #include "sanitizer_common/sanitizer_common_syscalls.inc" #include "sanitizer_common/sanitizer_syscalls_netbsd.inc" #ifdef NEED_TLS_GET_ADDR static void handle_tls_addr(void *arg, void *res) { … } #if !SANITIZER_S390 // Define own interceptor instead of sanitizer_common's for three reasons: // 1. It must not process pending signals. // Signal handlers may contain MOVDQA instruction (see below). // 2. It must be as simple as possible to not contain MOVDQA. // 3. Sanitizer_common version uses COMMON_INTERCEPTOR_INITIALIZE_RANGE which // is empty for tsan (meant only for msan). // Note: __tls_get_addr can be called with mis-aligned stack due to: // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066 // So the interceptor must work with mis-aligned stack, in particular, does not // execute MOVDQA with stack addresses. TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) { … } #else // SANITIZER_S390 TSAN_INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) { uptr res = __tls_get_offset_wrapper(arg, REAL(__tls_get_offset)); char *tp = static_cast<char *>(__builtin_thread_pointer()); handle_tls_addr(arg, res + tp); return res; } #endif #endif #if SANITIZER_NETBSD TSAN_INTERCEPTOR(void, _lwp_exit) { SCOPED_TSAN_INTERCEPTOR(_lwp_exit); DestroyThreadState(); REAL(_lwp_exit)(); } #define TSAN_MAYBE_INTERCEPT__LWP_EXIT … #else #define TSAN_MAYBE_INTERCEPT__LWP_EXIT #endif #if SANITIZER_FREEBSD TSAN_INTERCEPTOR(void, thr_exit, tid_t *state) { SCOPED_TSAN_INTERCEPTOR(thr_exit, state); DestroyThreadState(); REAL(thr_exit(state)); } #define TSAN_MAYBE_INTERCEPT_THR_EXIT … #else #define TSAN_MAYBE_INTERCEPT_THR_EXIT #endif TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_FREEBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS(…) TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(…) TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(…) namespace __tsan { static void finalize(void *arg) { … } #if !SANITIZER_APPLE && !SANITIZER_ANDROID static void unreachable() { … } #endif // Define default implementation since interception of libdispatch is optional. SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() { … } void InitializeInterceptors() { … } } // namespace __tsan // Invisible barrier for tests. // There were several unsuccessful iterations for this functionality: // 1. Initially it was implemented in user code using // REAL(pthread_barrier_wait). But pthread_barrier_wait is not supported on // MacOS. Futexes are linux-specific for this matter. // 2. Then we switched to atomics+usleep(10). But usleep produced parasitic // "as-if synchronized via sleep" messages in reports which failed some // output tests. // 3. Then we switched to atomics+sched_yield. But this produced tons of tsan- // visible events, which lead to "failed to restore stack trace" failures. // Note that no_sanitize_thread attribute does not turn off atomic interception // so attaching it to the function defined in user code does not help. // That's why we now have what we have. constexpr u32 kBarrierThreadBits = …; constexpr u32 kBarrierThreads = …; extern "C" { SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_init( atomic_uint32_t *barrier, u32 num_threads) { … } static u32 barrier_epoch(u32 value) { … } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait( atomic_uint32_t *barrier) { … } } // extern "C"
Codebase Browser
llvm