commit e99c11856df5dafb54e92af12c0858d187dc2ad2 (tree) parent d67d52abe57b13517013e81562d64bb7c2f77775 Author: Alex Rønne Petersen <alex@alexrp.com> Date: Wed, 5 Feb 2025 13:36:56 +0100 libtsan: Update to LLVM 20. Diffstat:
83 files changed, 3243 insertions(+), 2047 deletions(-)
diff --git a/lib/tsan/builtins/assembly.h b/lib/tsan/builtins/assembly.h @@ -290,4 +290,16 @@ CFI_END #endif +#ifdef __arm__ +#include "int_endianness.h" + +#if _YUGA_BIG_ENDIAN +#define VMOV_TO_DOUBLE(dst, src0, src1) vmov dst, src1, src0 SEPARATOR +#define VMOV_FROM_DOUBLE(dst0, dst1, src) vmov dst1, dst0, src SEPARATOR +#else +#define VMOV_TO_DOUBLE(dst, src0, src1) vmov dst, src0, src1 SEPARATOR +#define VMOV_FROM_DOUBLE(dst0, dst1, src) vmov dst0, dst1, src SEPARATOR +#endif +#endif + #endif // COMPILERRT_ASSEMBLY_H diff --git a/lib/tsan/interception/interception.h b/lib/tsan/interception/interception.h @@ -25,8 +25,19 @@ // These typedefs should be used only in the interceptor definitions to replace // the standard system types (e.g. SSIZE_T instead of ssize_t) -typedef __sanitizer::uptr SIZE_T; -typedef __sanitizer::sptr SSIZE_T; +// On Windows the system headers (basetsd.h) provide a conflicting definition +// of SIZE_T/SSIZE_T that do not match the real size_t/ssize_t for 32-bit +// systems (using long instead of the expected int). Work around the typedef +// redefinition by #defining SIZE_T instead of using a typedef. +// TODO: We should be using __sanitizer::usize (and a new ssize) instead of +// these new macros as long as we ensure they match the real system definitions. +#if SANITIZER_WINDOWS +// Ensure that (S)SIZE_T were already defined as we are about to override them. +# include <basetsd.h> +#endif + +#define SIZE_T __sanitizer::usize +#define SSIZE_T __sanitizer::ssize typedef __sanitizer::sptr PTRDIFF_T; typedef __sanitizer::s64 INTMAX_T; typedef __sanitizer::u64 UINTMAX_T; @@ -338,16 +349,8 @@ const interpose_substitution substitution_##func_name[] \ #endif // ISO C++ forbids casting between pointer-to-function and pointer-to-object, -// so we use casting via an integral type __interception::uptr, -// assuming that system is POSIX-compliant. Using other hacks seem -// challenging, as we don't even pass function type to -// INTERCEPT_FUNCTION macro, only its name. +// so we use casts via uintptr_t (the local __sanitizer::uptr equivalent). namespace __interception { -#if defined(_WIN64) -typedef unsigned long long uptr; -#else -typedef unsigned long uptr; -#endif // _WIN64 #if defined(__ELF__) && !SANITIZER_FUCHSIA // The use of interceptors makes many sanitizers unusable for static linking. diff --git a/lib/tsan/interception/interception_type_test.cpp b/lib/tsan/interception/interception_type_test.cpp @@ -12,28 +12,35 @@ //===----------------------------------------------------------------------===// #include "interception.h" +#include "sanitizer_common/sanitizer_type_traits.h" -#if SANITIZER_LINUX || SANITIZER_APPLE - -#include <sys/types.h> +#if __has_include(<sys/types.h>) +# include <sys/types.h> +#endif #include <stddef.h> #include <stdint.h> -COMPILER_CHECK(sizeof(::SIZE_T) == sizeof(size_t)); -COMPILER_CHECK(sizeof(::SSIZE_T) == sizeof(ssize_t)); -COMPILER_CHECK(sizeof(::PTRDIFF_T) == sizeof(ptrdiff_t)); +COMPILER_CHECK((__sanitizer::is_same<__sanitizer::uptr, ::uintptr_t>::value)); +COMPILER_CHECK((__sanitizer::is_same<__sanitizer::sptr, ::intptr_t>::value)); +COMPILER_CHECK((__sanitizer::is_same<__sanitizer::usize, ::size_t>::value)); +COMPILER_CHECK((__sanitizer::is_same<::PTRDIFF_T, ::ptrdiff_t>::value)); +COMPILER_CHECK((__sanitizer::is_same<::SIZE_T, ::size_t>::value)); +#if !SANITIZER_WINDOWS +// No ssize_t on Windows. +COMPILER_CHECK((__sanitizer::is_same<::SSIZE_T, ::ssize_t>::value)); +#endif +// TODO: These are not actually the same type on Linux (long vs long long) COMPILER_CHECK(sizeof(::INTMAX_T) == sizeof(intmax_t)); +COMPILER_CHECK(sizeof(::UINTMAX_T) == sizeof(uintmax_t)); -# if SANITIZER_GLIBC || SANITIZER_ANDROID +#if SANITIZER_GLIBC || SANITIZER_ANDROID COMPILER_CHECK(sizeof(::OFF64_T) == sizeof(off64_t)); -# endif +#endif // The following are the cases when pread (and friends) is used instead of // pread64. In those cases we need OFF_T to match off_t. We don't care about the // rest (they depend on _FILE_OFFSET_BITS setting when building an application). -# if SANITIZER_ANDROID || !defined _FILE_OFFSET_BITS || \ - _FILE_OFFSET_BITS != 64 +#if !SANITIZER_WINDOWS && (SANITIZER_ANDROID || !defined _FILE_OFFSET_BITS || \ + _FILE_OFFSET_BITS != 64) COMPILER_CHECK(sizeof(::OFF_T) == sizeof(off_t)); -# endif - #endif diff --git a/lib/tsan/interception/interception_win.cpp b/lib/tsan/interception/interception_win.cpp @@ -27,7 +27,7 @@ // // 1) Detour // -// The Detour hooking technique is assuming the presence of an header with +// The Detour hooking technique is assuming the presence of a header with // padding and an overridable 2-bytes nop instruction (mov edi, edi). The // nop instruction can safely be replaced by a 2-bytes jump without any need // to save the instruction. A jump to the target is encoded in the function @@ -47,7 +47,7 @@ // // func: jmp <label> --> func: jmp <hook> // -// On an 64-bit architecture, a trampoline is inserted. +// On a 64-bit architecture, a trampoline is inserted. // // func: jmp <label> --> func: jmp <tramp> // [...] @@ -60,7 +60,7 @@ // // 3) HotPatch // -// The HotPatch hooking is assuming the presence of an header with padding +// The HotPatch hooking is assuming the presence of a header with padding // and a first instruction with at least 2-bytes. // // The reason to enforce the 2-bytes limitation is to provide the minimal @@ -80,7 +80,7 @@ // real: <instr> // jmp <body> // -// On an 64-bit architecture: +// On a 64-bit architecture: // // head: 6 x nop head: jmp QWORD [addr1] // func: <instr> --> func: jmp short <head> @@ -110,7 +110,7 @@ // <instr> // jmp <body> // -// On an 64-bit architecture: +// On a 64-bit architecture: // // func: <instr> --> func: jmp QWORD [addr1] // <instr> @@ -130,6 +130,7 @@ #include "sanitizer_common/sanitizer_platform.h" #define WIN32_LEAN_AND_MEAN #include <windows.h> +#include <psapi.h> namespace __interception { @@ -186,8 +187,12 @@ static uptr GetMmapGranularity() { return si.dwAllocationGranularity; } +UNUSED static uptr RoundDownTo(uptr size, uptr boundary) { + return size & ~(boundary - 1); +} + UNUSED static uptr RoundUpTo(uptr size, uptr boundary) { - return (size + boundary - 1) & ~(boundary - 1); + return RoundDownTo(size + boundary - 1, boundary); } // FIXME: internal_str* and internal_mem* functions should be moved from the @@ -208,6 +213,18 @@ static char* _strchr(char* str, char c) { return nullptr; } +static int _strcmp(const char *s1, const char *s2) { + while (true) { + unsigned c1 = *s1; + unsigned c2 = *s2; + if (c1 != c2) return (c1 < c2) ? -1 : 1; + if (c1 == 0) break; + s1++; + s2++; + } + return 0; +} + static void _memset(void *p, int value, size_t sz) { for (size_t i = 0; i < sz; ++i) ((char*)p)[i] = (char)value; @@ -284,8 +301,11 @@ static void WriteJumpInstruction(uptr from, uptr target) { static void WriteShortJumpInstruction(uptr from, uptr target) { sptr offset = target - from - kShortJumpInstructionLength; - if (offset < -128 || offset > 127) + if (offset < -128 || offset > 127) { + ReportError("interception_win: cannot write short jmp from %p to %p\n", + (void *)from, (void *)target); InterceptionFailed(); + } *(u8*)from = 0xEB; *(u8*)(from + 1) = (u8)offset; } @@ -339,32 +359,78 @@ struct TrampolineMemoryRegion { uptr max_size; }; -UNUSED static const uptr kTrampolineScanLimitRange = 1ull << 31; // 2 gig +UNUSED static const uptr kTrampolineRangeLimit = 1ull << 31; // 2 gig static const int kMaxTrampolineRegion = 1024; static TrampolineMemoryRegion TrampolineRegions[kMaxTrampolineRegion]; -static void *AllocateTrampolineRegion(uptr image_address, size_t granularity) { -#if SANITIZER_WINDOWS64 - uptr address = image_address; - uptr scanned = 0; - while (scanned < kTrampolineScanLimitRange) { +static void *AllocateTrampolineRegion(uptr min_addr, uptr max_addr, + uptr func_addr, size_t granularity) { +# if SANITIZER_WINDOWS64 + // Clamp {min,max}_addr to the accessible address space. + SYSTEM_INFO system_info; + ::GetSystemInfo(&system_info); + uptr min_virtual_addr = + RoundUpTo((uptr)system_info.lpMinimumApplicationAddress, granularity); + uptr max_virtual_addr = + RoundDownTo((uptr)system_info.lpMaximumApplicationAddress, granularity); + if (min_addr < min_virtual_addr) + min_addr = min_virtual_addr; + if (max_addr > max_virtual_addr) + max_addr = max_virtual_addr; + + // This loop probes the virtual address space to find free memory in the + // [min_addr, max_addr] interval. The search starts from func_addr and + // proceeds "outwards" towards the interval bounds using two probes, lo_addr + // and hi_addr, for addresses lower/higher than func_addr. At each step, it + // considers the probe closest to func_addr. If that address is not free, the + // probe is advanced (lower or higher depending on the probe) to the next + // memory block and the search continues. + uptr lo_addr = RoundDownTo(func_addr, granularity); + uptr hi_addr = RoundUpTo(func_addr, granularity); + while (lo_addr >= min_addr || hi_addr <= max_addr) { + // Consider the in-range address closest to func_addr. + uptr addr; + if (lo_addr < min_addr) + addr = hi_addr; + else if (hi_addr > max_addr) + addr = lo_addr; + else + addr = (hi_addr - func_addr < func_addr - lo_addr) ? hi_addr : lo_addr; + MEMORY_BASIC_INFORMATION info; - if (!::VirtualQuery((void*)address, &info, sizeof(info))) + if (!::VirtualQuery((void *)addr, &info, sizeof(info))) { + ReportError( + "interception_win: VirtualQuery in AllocateTrampolineRegion failed " + "for %p\n", + (void *)addr); return nullptr; + } - // Check whether a region can be allocated at |address|. + // Check whether a region can be allocated at |addr|. if (info.State == MEM_FREE && info.RegionSize >= granularity) { - void *page = ::VirtualAlloc((void*)RoundUpTo(address, granularity), - granularity, - MEM_RESERVE | MEM_COMMIT, - PAGE_EXECUTE_READWRITE); + void *page = + ::VirtualAlloc((void *)addr, granularity, MEM_RESERVE | MEM_COMMIT, + PAGE_EXECUTE_READWRITE); + if (page == nullptr) + ReportError( + "interception_win: VirtualAlloc in AllocateTrampolineRegion failed " + "for %p\n", + (void *)addr); return page; } - // Move to the next region. - address = (uptr)info.BaseAddress + info.RegionSize; - scanned += info.RegionSize; + if (addr == lo_addr) + lo_addr = + RoundDownTo((uptr)info.AllocationBase - granularity, granularity); + if (addr == hi_addr) + hi_addr = + RoundUpTo((uptr)info.BaseAddress + info.RegionSize, granularity); } + + ReportError( + "interception_win: AllocateTrampolineRegion failed to find free memory; " + "min_addr: %p, max_addr: %p, func_addr: %p, granularity: %zu\n", + (void *)min_addr, (void *)max_addr, (void *)func_addr, granularity); return nullptr; #else return ::VirtualAlloc(nullptr, @@ -385,15 +451,51 @@ void TestOnlyReleaseTrampolineRegions() { } } -static uptr AllocateMemoryForTrampoline(uptr image_address, size_t size) { - // Find a region within 2G with enough space to allocate |size| bytes. +static uptr AllocateMemoryForTrampoline(uptr func_address, size_t size) { +# if SANITIZER_WINDOWS64 + uptr min_addr = func_address - kTrampolineRangeLimit; + uptr max_addr = func_address + kTrampolineRangeLimit - size; + + // Allocate memory within 2GB of the module (DLL or EXE file) so that any + // address within the module can be referenced with PC-relative operands. + // This allows us to not just jump to the trampoline with a PC-relative + // offset, but to relocate any instructions that we copy to the trampoline + // which have references to the original module. If we can't find the base + // address of the module (e.g. if func_address is in mmap'ed memory), just + // stay within 2GB of func_address. + HMODULE module; + if (::GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCWSTR)func_address, &module)) { + MODULEINFO module_info; + if (::GetModuleInformation(::GetCurrentProcess(), module, + &module_info, sizeof(module_info))) { + min_addr = (uptr)module_info.lpBaseOfDll + module_info.SizeOfImage - + kTrampolineRangeLimit; + max_addr = (uptr)module_info.lpBaseOfDll + kTrampolineRangeLimit - size; + } + } + + // Check for overflow. + if (min_addr > func_address) + min_addr = 0; + if (max_addr < func_address) + max_addr = ~(uptr)0; +# else + uptr min_addr = 0; + uptr max_addr = ~min_addr; +# endif + + // Find a region within [min_addr,max_addr] with enough space to allocate + // |size| bytes. TrampolineMemoryRegion *region = nullptr; for (size_t bucket = 0; bucket < kMaxTrampolineRegion; ++bucket) { TrampolineMemoryRegion* current = &TrampolineRegions[bucket]; if (current->content == 0) { // No valid region found, allocate a new region. size_t bucket_size = GetMmapGranularity(); - void *content = AllocateTrampolineRegion(image_address, bucket_size); + void *content = AllocateTrampolineRegion(min_addr, max_addr, func_address, + bucket_size); if (content == nullptr) return 0U; @@ -403,13 +505,9 @@ static uptr AllocateMemoryForTrampoline(uptr image_address, size_t size) { region = current; break; } else if (current->max_size - current->allocated_size > size) { -#if SANITIZER_WINDOWS64 - // In 64-bits, the memory space must be allocated within 2G boundary. - uptr next_address = current->content + current->allocated_size; - if (next_address < image_address || - next_address - image_address >= 0x7FFF0000) - continue; -#endif + uptr next_address = current->content + current->allocated_size; + if (next_address < min_addr || next_address > max_addr) + continue; // The space can be allocated in the current region. region = current; break; @@ -458,6 +556,10 @@ static const u8 kPrologueWithShortJump2[] = { // Returns 0 on error. static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { + if (rel_offset) { + *rel_offset = 0; + } + #if SANITIZER_ARM64 // An ARM64 instruction is 4 bytes long. return 4; @@ -497,8 +599,14 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0x6A: // 6A XX = push XX return 2; + // This instruction can be encoded with a 16-bit immediate but that is + // incredibly unlikely. + case 0x68: // 68 XX XX XX XX : push imm32 + return 5; + case 0xb8: // b8 XX XX XX XX : mov eax, XX XX XX XX case 0xB9: // b9 XX XX XX XX : mov ecx, XX XX XX XX + case 0xBA: // ba XX XX XX XX : mov edx, XX XX XX XX return 5; // Cannot overwrite control-instruction. Return 0 to indicate failure. @@ -529,19 +637,43 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0xFF8B: // 8B FF : mov edi, edi case 0xEC8B: // 8B EC : mov ebp, esp case 0xc889: // 89 C8 : mov eax, ecx + case 0xD189: // 89 D1 : mov ecx, edx case 0xE589: // 89 E5 : mov ebp, esp case 0xC18B: // 8B C1 : mov eax, ecx + case 0xC031: // 31 C0 : xor eax, eax + case 0xC931: // 31 C9 : xor ecx, ecx + case 0xD231: // 31 D2 : xor edx, edx case 0xC033: // 33 C0 : xor eax, eax case 0xC933: // 33 C9 : xor ecx, ecx case 0xD233: // 33 D2 : xor edx, edx + case 0xDB84: // 84 DB : test bl,bl + case 0xC084: // 84 C0 : test al,al + case 0xC984: // 84 C9 : test cl,cl + case 0xD284: // 84 D2 : test dl,dl return 2; + case 0x3980: // 80 39 XX : cmp BYTE PTR [rcx], XX + case 0x4D8B: // 8B 4D XX : mov XX(%ebp), ecx + case 0x558B: // 8B 55 XX : mov XX(%ebp), edx + case 0x758B: // 8B 75 XX : mov XX(%ebp), esp + case 0xE483: // 83 E4 XX : and esp, XX + case 0xEC83: // 83 EC XX : sub esp, XX + case 0xC1F6: // F6 C1 XX : test cl, XX + return 3; + + case 0x89FF: // FF 89 XX XX XX XX : dec dword ptr [ecx + XX XX XX XX] + case 0xEC81: // 81 EC XX XX XX XX : sub esp, XX XX XX XX + return 6; + // Cannot overwrite control-instruction. Return 0 to indicate failure. - case 0x25FF: // FF 25 XX XX XX XX : jmp [XXXXXXXX] + case 0x25FF: // FF 25 XX YY ZZ WW : jmp dword ptr ds:[WWZZYYXX] return 0; } - switch (0x00FFFFFF & *(u32*)address) { + switch (0x00FFFFFF & *(u32 *)address) { + case 0x244C8D: // 8D 4C 24 XX : lea ecx, [esp + XX] + case 0x2474FF: // FF 74 24 XX : push qword ptr [rsp + XX] + return 4; case 0x24A48D: // 8D A4 24 XX XX XX XX : lea esp, [esp + XX XX XX XX] return 7; } @@ -556,6 +688,21 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0xA1: // A1 XX XX XX XX XX XX XX XX : // movabs eax, dword ptr ds:[XXXXXXXX] return 9; + case 0xF2: + switch (*(u32 *)(address + 1)) { + case 0x2444110f: // f2 0f 11 44 24 XX movsd QWORD PTR + // [rsp + XX], xmm0 + case 0x244c110f: // f2 0f 11 4c 24 XX movsd QWORD PTR + // [rsp + XX], xmm1 + case 0x2454110f: // f2 0f 11 54 24 XX movsd QWORD PTR + // [rsp + XX], xmm2 + case 0x245c110f: // f2 0f 11 5c 24 XX movsd QWORD PTR + // [rsp + XX], xmm3 + case 0x2464110f: // f2 0f 11 64 24 XX movsd QWORD PTR + // [rsp + XX], xmm4 + return 6; + } + break; case 0x83: const u8 next_byte = *(u8*)(address + 1); @@ -584,55 +731,155 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0x018a: // mov al, byte ptr [rcx] return 2; + case 0x7E80: // 80 7E YY XX cmp BYTE PTR [rsi+YY], XX + case 0x7D80: // 80 7D YY XX cmp BYTE PTR [rbp+YY], XX + case 0x7A80: // 80 7A YY XX cmp BYTE PTR [rdx+YY], XX + case 0x7880: // 80 78 YY XX cmp BYTE PTR [rax+YY], XX + case 0x7B80: // 80 7B YY XX cmp BYTE PTR [rbx+YY], XX + case 0x7980: // 80 79 YY XX cmp BYTE ptr [rcx+YY], XX + return 4; + case 0x058A: // 8A 05 XX XX XX XX : mov al, byte ptr [XX XX XX XX] case 0x058B: // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX] if (rel_offset) *rel_offset = 2; + case 0xB841: // 41 B8 XX XX XX XX : mov r8d, XX XX XX XX return 6; + + case 0x7E81: // 81 7E YY XX XX XX XX cmp DWORD PTR [rsi+YY], XX XX XX XX + case 0x7D81: // 81 7D YY XX XX XX XX cmp DWORD PTR [rbp+YY], XX XX XX XX + case 0x7A81: // 81 7A YY XX XX XX XX cmp DWORD PTR [rdx+YY], XX XX XX XX + case 0x7881: // 81 78 YY XX XX XX XX cmp DWORD PTR [rax+YY], XX XX XX XX + case 0x7B81: // 81 7B YY XX XX XX XX cmp DWORD PTR [rbx+YY], XX XX XX XX + case 0x7981: // 81 79 YY XX XX XX XX cmp dword ptr [rcx+YY], XX XX XX XX + return 7; } - switch (0x00FFFFFF & *(u32*)address) { - case 0xe58948: // 48 8b c4 : mov rbp, rsp - case 0xc18b48: // 48 8b c1 : mov rax, rcx - case 0xc48b48: // 48 8b c4 : mov rax, rsp - case 0xd9f748: // 48 f7 d9 : neg rcx - case 0xd12b48: // 48 2b d1 : sub rdx, rcx - case 0x07c1f6: // f6 c1 07 : test cl, 0x7 - case 0xc98548: // 48 85 C9 : test rcx, rcx - case 0xd28548: // 48 85 d2 : test rdx, rdx + switch (0x00FFFFFF & *(u32 *)address) { + case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax] + case 0xc00b4d: // 4d 0b c0 : or r8, r8 + case 0xc03345: // 45 33 c0 : xor r8d, r8d + case 0xc08548: // 48 85 c0 : test rax, rax case 0xc0854d: // 4d 85 c0 : test r8, r8 + case 0xc08b41: // 41 8b c0 : mov eax, r8d + case 0xc0ff48: // 48 ff c0 : inc rax + case 0xc0ff49: // 49 ff c0 : inc r8 + case 0xc18b41: // 41 8b c1 : mov eax, r9d + case 0xc18b48: // 48 8b c1 : mov rax, rcx + case 0xc18b4c: // 4c 8b c1 : mov r8, rcx + case 0xc1ff48: // 48 ff c1 : inc rcx + case 0xc1ff49: // 49 ff c1 : inc r9 + case 0xc28b41: // 41 8b c2 : mov eax, r10d + case 0x01b60f: // 0f b6 01 : movzx eax, BYTE PTR [rcx] + case 0x09b60f: // 0f b6 09 : movzx ecx, BYTE PTR [rcx] + case 0x11b60f: // 0f b6 11 : movzx edx, BYTE PTR [rcx] case 0xc2b60f: // 0f b6 c2 : movzx eax, dl - case 0xc03345: // 45 33 c0 : xor r8d, r8d + case 0xc2ff48: // 48 ff c2 : inc rdx + case 0xc2ff49: // 49 ff c2 : inc r10 + case 0xc38b41: // 41 8b c3 : mov eax, r11d + case 0xc3ff48: // 48 ff c3 : inc rbx + case 0xc3ff49: // 49 ff c3 : inc r11 + case 0xc48b41: // 41 8b c4 : mov eax, r12d + case 0xc48b48: // 48 8b c4 : mov rax, rsp + case 0xc4ff49: // 49 ff c4 : inc r12 + case 0xc5ff49: // 49 ff c5 : inc r13 + case 0xc6ff48: // 48 ff c6 : inc rsi + case 0xc6ff49: // 49 ff c6 : inc r14 + case 0xc7ff48: // 48 ff c7 : inc rdi + case 0xc7ff49: // 49 ff c7 : inc r15 case 0xc93345: // 45 33 c9 : xor r9d, r9d - case 0xdb3345: // 45 33 DB : xor r11d, r11d - case 0xd98b4c: // 4c 8b d9 : mov r11, rcx - case 0xd28b4c: // 4c 8b d2 : mov r10, rdx - case 0xc98b4c: // 4C 8B C9 : mov r9, rcx - case 0xc18b4c: // 4C 8B C1 : mov r8, rcx - case 0xd2b60f: // 0f b6 d2 : movzx edx, dl + case 0xc98548: // 48 85 c9 : test rcx, rcx + case 0xc9854d: // 4d 85 c9 : test r9, r9 + case 0xc98b4c: // 4c 8b c9 : mov r9, rcx + case 0xd12948: // 48 29 d1 : sub rcx, rdx case 0xca2b48: // 48 2b ca : sub rcx, rdx case 0xca3b48: // 48 3b ca : cmp rcx, rdx - case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax] - case 0xc00b4d: // 3d 0b c0 : or r8, r8 - case 0xc08b41: // 41 8b c0 : mov eax, r8d + case 0xd12b48: // 48 2b d1 : sub rdx, rcx case 0xd18b48: // 48 8b d1 : mov rdx, rcx - case 0xdc8b4c: // 4c 8b dc : mov r11, rsp case 0xd18b4c: // 4c 8b d1 : mov r10, rcx - case 0xE0E483: // 83 E4 E0 : and esp, 0xFFFFFFE0 + case 0xd28548: // 48 85 d2 : test rdx, rdx + case 0xd2854d: // 4d 85 d2 : test r10, r10 + case 0xd28b4c: // 4c 8b d2 : mov r10, rdx + case 0xd2b60f: // 0f b6 d2 : movzx edx, dl + case 0xd2be0f: // 0f be d2 : movsx edx, dl + case 0xd98b4c: // 4c 8b d9 : mov r11, rcx + case 0xd9f748: // 48 f7 d9 : neg rcx + case 0xc03145: // 45 31 c0 : xor r8d,r8d + case 0xc93145: // 45 31 c9 : xor r9d,r9d + case 0xdb3345: // 45 33 db : xor r11d, r11d + case 0xc08445: // 45 84 c0 : test r8b,r8b + case 0xd28445: // 45 84 d2 : test r10b,r10b + case 0xdb8548: // 48 85 db : test rbx, rbx + case 0xdb854d: // 4d 85 db : test r11, r11 + case 0xdc8b4c: // 4c 8b dc : mov r11, rsp + case 0xe48548: // 48 85 e4 : test rsp, rsp + case 0xe4854d: // 4d 85 e4 : test r12, r12 + case 0xc88948: // 48 89 c8 : mov rax,rcx + case 0xcb8948: // 48 89 cb : mov rbx,rcx + case 0xd08948: // 48 89 d0 : mov rax,rdx + case 0xd18948: // 48 89 d1 : mov rcx,rdx + case 0xd38948: // 48 89 d3 : mov rbx,rdx + case 0xe58948: // 48 89 e5 : mov rbp, rsp + case 0xed8548: // 48 85 ed : test rbp, rbp + case 0xc88949: // 49 89 c8 : mov r8, rcx + case 0xc98949: // 49 89 c9 : mov r9, rcx + case 0xca8949: // 49 89 ca : mov r10,rcx + case 0xd08949: // 49 89 d0 : mov r8, rdx + case 0xd18949: // 49 89 d1 : mov r9, rdx + case 0xd28949: // 49 89 d2 : mov r10, rdx + case 0xd38949: // 49 89 d3 : mov r11, rdx + case 0xed854d: // 4d 85 ed : test r13, r13 + case 0xf6854d: // 4d 85 f6 : test r14, r14 + case 0xff854d: // 4d 85 ff : test r15, r15 return 3; + case 0x245489: // 89 54 24 XX : mov DWORD PTR[rsp + XX], edx + case 0x428d44: // 44 8d 42 XX : lea r8d , [rdx + XX] + case 0x588948: // 48 89 58 XX : mov QWORD PTR[rax + XX], rbx case 0xec8348: // 48 83 ec XX : sub rsp, XX case 0xf88349: // 49 83 f8 XX : cmp r8, XX - case 0x588948: // 48 89 58 XX : mov QWORD PTR[rax + XX], rbx + case 0x488d49: // 49 8d 48 XX : lea rcx, [...] + case 0x048d4c: // 4c 8d 04 XX : lea r8, [...] + case 0x148d4e: // 4e 8d 14 XX : lea r10, [...] + case 0x398366: // 66 83 39 XX : cmp WORD PTR [rcx], XX return 4; + case 0x441F0F: // 0F 1F 44 XX XX : nop DWORD PTR [...] + case 0x246483: // 83 64 24 XX YY : and DWORD PTR [rsp+XX], YY + return 5; + + case 0x788166: // 66 81 78 XX YY YY cmp WORD PTR [rax+XX], YY YY + case 0x798166: // 66 81 79 XX YY YY cmp WORD PTR [rcx+XX], YY YY + case 0x7a8166: // 66 81 7a XX YY YY cmp WORD PTR [rdx+XX], YY YY + case 0x7b8166: // 66 81 7b XX YY YY cmp WORD PTR [rbx+XX], YY YY + case 0x7e8166: // 66 81 7e XX YY YY cmp WORD PTR [rsi+XX], YY YY + case 0x7f8166: // 66 81 7f XX YY YY cmp WORD PTR [rdi+XX], YY YY + return 6; + case 0xec8148: // 48 81 EC XX XX XX XX : sub rsp, XXXXXXXX + case 0xc0c748: // 48 C7 C0 XX XX XX XX : mov rax, XX XX XX XX return 7; + // clang-format off + case 0x788141: // 41 81 78 XX YY YY YY YY : cmp DWORD PTR [r8+YY], XX XX XX XX + case 0x798141: // 41 81 79 XX YY YY YY YY : cmp DWORD PTR [r9+YY], XX XX XX XX + case 0x7a8141: // 41 81 7a XX YY YY YY YY : cmp DWORD PTR [r10+YY], XX XX XX XX + case 0x7b8141: // 41 81 7b XX YY YY YY YY : cmp DWORD PTR [r11+YY], XX XX XX XX + case 0x7d8141: // 41 81 7d XX YY YY YY YY : cmp DWORD PTR [r13+YY], XX XX XX XX + case 0x7e8141: // 41 81 7e XX YY YY YY YY : cmp DWORD PTR [r14+YY], XX XX XX XX + case 0x7f8141: // 41 81 7f YY XX XX XX XX : cmp DWORD PTR [r15+YY], XX XX XX XX + case 0x247c81: // 81 7c 24 YY XX XX XX XX : cmp DWORD PTR [rsp+YY], XX XX XX XX + return 8; + // clang-format on + case 0x058b48: // 48 8b 05 XX XX XX XX : // mov rax, QWORD PTR [rip + XXXXXXXX] case 0x058d48: // 48 8d 05 XX XX XX XX : // lea rax, QWORD PTR [rip + XXXXXXXX] + case 0x0d8948: // 48 89 0d XX XX XX XX : + // mov QWORD PTR [rip + XXXXXXXX], rcx + case 0x158948: // 48 89 15 XX XX XX XX : + // mov QWORD PTR [rip + XXXXXXXX], rdx case 0x25ff48: // 48 ff 25 XX XX XX XX : // rex.W jmp QWORD PTR [rip + XXXXXXXX] case 0x158D4C: // 4c 8d 15 XX XX XX XX : lea r10, [rip + XX] @@ -644,9 +891,19 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0x2444c7: // C7 44 24 XX YY YY YY YY // mov dword ptr [rsp + XX], YYYYYYYY return 8; + + case 0x7c8141: // 41 81 7c ZZ YY XX XX XX XX + // cmp DWORD PTR [reg+reg*n+YY], XX XX XX XX + return 9; } switch (*(u32*)(address)) { + case 0x01b60f44: // 44 0f b6 01 : movzx r8d, BYTE PTR [rcx] + case 0x09b60f44: // 44 0f b6 09 : movzx r9d, BYTE PTR [rcx] + case 0x0ab60f44: // 44 0f b6 0a : movzx r8d, BYTE PTR [rdx] + case 0x11b60f44: // 44 0f b6 11 : movzx r10d, BYTE PTR [rcx] + case 0x1ab60f44: // 44 0f b6 1a : movzx r11d, BYTE PTR [rdx] + return 4; case 0x24448b48: // 48 8b 44 24 XX : mov rax, QWORD ptr [rsp + XX] case 0x246c8948: // 48 89 6C 24 XX : mov QWORD ptr [rsp + XX], rbp case 0x245c8948: // 48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx @@ -656,9 +913,19 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0x24548948: // 48 89 54 24 XX : mov QWORD PTR [rsp + XX], rdx case 0x244c894c: // 4c 89 4c 24 XX : mov QWORD PTR [rsp + XX], r9 case 0x2444894c: // 4c 89 44 24 XX : mov QWORD PTR [rsp + XX], r8 + case 0x244c8944: // 44 89 4c 24 XX mov DWORD PTR [rsp + XX], r9d + case 0x24448944: // 44 89 44 24 XX mov DWORD PTR [rsp + XX], r8d + case 0x246c8d48: // 48 8d 6c 24 XX : lea rbp, [rsp + XX] return 5; - case 0x24648348: // 48 83 64 24 XX : and QWORD PTR [rsp + XX], YY + case 0x24648348: // 48 83 64 24 XX YY : and QWORD PTR [rsp + XX], YY return 6; + case 0x24A48D48: // 48 8D A4 24 XX XX XX XX : lea rsp, [rsp + XX XX XX XX] + return 8; + } + + switch (0xFFFFFFFFFFULL & *(u64 *)(address)) { + case 0xC07E0F4866: // 66 48 0F 7E C0 : movq rax, xmm0 + return 5; } #else @@ -671,11 +938,10 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0x458B: // 8B 45 XX : mov eax, dword ptr [ebp + XX] case 0x5D8B: // 8B 5D XX : mov ebx, dword ptr [ebp + XX] case 0x7D8B: // 8B 7D XX : mov edi, dword ptr [ebp + XX] - case 0xEC83: // 83 EC XX : sub esp, XX + case 0x758B: // 8B 75 XX : mov esi, dword ptr [ebp + XX] case 0x75FF: // FF 75 XX : push dword ptr [ebp + XX] return 3; case 0xC1F7: // F7 C1 XX YY ZZ WW : test ecx, WWZZYYXX - case 0x25FF: // FF 25 XX YY ZZ WW : jmp dword ptr ds:[WWZZYYXX] return 6; case 0x3D83: // 83 3D XX YY ZZ WW TT : cmp TT, WWZZYYXX return 7; @@ -718,6 +984,10 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { return 0; } +size_t TestOnlyGetInstructionSize(uptr address, size_t *rel_offset) { + return GetInstructionSize(address, rel_offset); +} + // Returns 0 on error. static size_t RoundUpToInstrBoundary(size_t size, uptr address) { size_t cursor = 0; @@ -745,8 +1015,14 @@ static bool CopyInstructions(uptr to, uptr from, size_t size) { // this will be untrue if relocated_offset \notin [-2**31, 2**31) s64 delta = to - from; s64 relocated_offset = *(s32 *)(to + cursor + rel_offset) - delta; - if (-0x8000'0000ll > relocated_offset || relocated_offset > 0x7FFF'FFFFll) + if (-0x8000'0000ll > relocated_offset || + relocated_offset > 0x7FFF'FFFFll) { + ReportError( + "interception_win: CopyInstructions relocated_offset %lld outside " + "32-bit range\n", + (long long)relocated_offset); return false; + } # else // on 32-bit, the relative offset will always be correct s32 delta = to - from; @@ -969,8 +1245,7 @@ static void **InterestingDLLsAvailable() { "libc++.dll", // libc++ "libunwind.dll", // libunwind # endif - // NTDLL should go last as it exports some functions that we should - // override in the CRT [presumably only used internally]. + // NTDLL must go last as it gets special treatment in OverrideFunction. "ntdll.dll", NULL }; @@ -1027,7 +1302,7 @@ uptr InternalGetProcAddress(void *module, const char *func_name) { for (DWORD i = 0; i < exports->NumberOfNames; i++) { RVAPtr<char> name(module, names[i]); - if (!strcmp(func_name, name)) { + if (!_strcmp(func_name, name)) { DWORD index = ordinals[i]; RVAPtr<char> func(module, functions[index]); @@ -1040,19 +1315,27 @@ uptr InternalGetProcAddress(void *module, const char *func_name) { // exported directory. char function_name[256]; size_t funtion_name_length = _strlen(func); - if (funtion_name_length >= sizeof(function_name) - 1) + if (funtion_name_length >= sizeof(function_name) - 1) { + ReportError("interception_win: func too long: '%s'\n", (char *)func); InterceptionFailed(); + } _memcpy(function_name, func, funtion_name_length); function_name[funtion_name_length] = '\0'; char* separator = _strchr(function_name, '.'); - if (!separator) + if (!separator) { + ReportError("interception_win: no separator in '%s'\n", + function_name); InterceptionFailed(); + } *separator = '\0'; void* redirected_module = GetModuleHandleA(function_name); - if (!redirected_module) + if (!redirected_module) { + ReportError("interception_win: GetModuleHandleA failed for '%s'\n", + function_name); InterceptionFailed(); + } return InternalGetProcAddress(redirected_module, separator + 1); } @@ -1065,9 +1348,22 @@ uptr InternalGetProcAddress(void *module, const char *func_name) { bool OverrideFunction( const char *func_name, uptr new_func, uptr *orig_old_func) { + static const char *kNtDllIgnore[] = { + "memcmp", "memcpy", "memmove", "memset" + }; + bool hooked = false; void **DLLs = InterestingDLLsAvailable(); for (size_t i = 0; DLLs[i]; ++i) { + if (DLLs[i + 1] == nullptr) { + // This is the last DLL, i.e. NTDLL. It exports some functions that + // we only want to override in the CRT. + for (const char *ignored : kNtDllIgnore) { + if (_strcmp(func_name, ignored) == 0) + return hooked; + } + } + uptr func_addr = InternalGetProcAddress(DLLs[i], func_name); if (func_addr && OverrideFunction(func_addr, new_func, orig_old_func)) { @@ -1121,7 +1417,7 @@ bool OverrideImportedFunction(const char *module_to_patch, RVAPtr<IMAGE_IMPORT_BY_NAME> import_by_name( module, name_table->u1.ForwarderString); const char *funcname = &import_by_name->Name[0]; - if (strcmp(funcname, function_name) == 0) + if (_strcmp(funcname, function_name) == 0) break; } } @@ -1144,4 +1440,4 @@ bool OverrideImportedFunction(const char *module_to_patch, } // namespace __interception -#endif // SANITIZER_APPLE +#endif // SANITIZER_WINDOWS diff --git a/lib/tsan/interception/interception_win.h b/lib/tsan/interception/interception_win.h @@ -63,6 +63,9 @@ bool OverrideFunctionWithTrampoline( // Exposed for unittests void TestOnlyReleaseTrampolineRegions(); +// Exposed for unittests +SIZE_T TestOnlyGetInstructionSize(uptr address, SIZE_T *rel_offset); + } // namespace __interception #if defined(INTERCEPTION_DYNAMIC_CRT) diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator.cpp b/lib/tsan/sanitizer_common/sanitizer_allocator.cpp @@ -59,7 +59,7 @@ static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache, static void *RawInternalRealloc(void *ptr, uptr size, InternalAllocatorCache *cache) { - uptr alignment = 8; + constexpr usize alignment = Max<usize>(8, sizeof(void *)); if (cache == 0) { SpinMutexLock l(&internal_allocator_cache_mu); return internal_allocator()->Reallocate(&internal_allocator_cache, ptr, @@ -137,7 +137,8 @@ void InternalAllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { } // LowLevelAllocator -constexpr uptr kLowLevelAllocatorDefaultAlignment = 8; +constexpr usize kLowLevelAllocatorDefaultAlignment = + Max<usize>(8, sizeof(void *)); constexpr uptr kMinNumPagesRounded = 16; constexpr uptr kMinRoundedSize = 65536; static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment; diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_dlsym.h b/lib/tsan/sanitizer_common/sanitizer_allocator_dlsym.h @@ -15,6 +15,8 @@ #define SANITIZER_ALLOCATOR_DLSYM_H #include "sanitizer_allocator_internal.h" +#include "sanitizer_common/sanitizer_allocator_checks.h" +#include "sanitizer_common/sanitizer_internal_defs.h" namespace __sanitizer { @@ -31,24 +33,22 @@ struct DlSymAllocator { UNLIKELY(internal_allocator()->FromPrimary(ptr)); } - static void *Allocate(uptr size_in_bytes) { - void *ptr = InternalAlloc(size_in_bytes, nullptr, kWordSize); + static void *Allocate(uptr size_in_bytes, uptr align = kWordSize) { + void *ptr = InternalAlloc(size_in_bytes, nullptr, align); CHECK(internal_allocator()->FromPrimary(ptr)); - Details::OnAllocate(ptr, - internal_allocator()->GetActuallyAllocatedSize(ptr)); + Details::OnAllocate(ptr, GetSize(ptr)); return ptr; } - static void *Callocate(SIZE_T nmemb, SIZE_T size) { + static void *Callocate(usize nmemb, usize size) { void *ptr = InternalCalloc(nmemb, size); CHECK(internal_allocator()->FromPrimary(ptr)); - Details::OnAllocate(ptr, - internal_allocator()->GetActuallyAllocatedSize(ptr)); + Details::OnAllocate(ptr, GetSize(ptr)); return ptr; } static void Free(void *ptr) { - uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr); + uptr size = GetSize(ptr); Details::OnFree(ptr, size); InternalFree(ptr); } @@ -61,7 +61,7 @@ struct DlSymAllocator { Free(ptr); return nullptr; } - uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr); + uptr size = GetSize(ptr); uptr memcpy_size = Min(new_size, size); void *new_ptr = Allocate(new_size); if (new_ptr) @@ -70,6 +70,15 @@ struct DlSymAllocator { return new_ptr; } + static void *ReallocArray(void *ptr, uptr count, uptr size) { + CHECK(!CheckForCallocOverflow(count, size)); + return Realloc(ptr, count * size); + } + + static uptr GetSize(void *ptr) { + return internal_allocator()->GetActuallyAllocatedSize(ptr); + } + static void OnAllocate(const void *ptr, uptr size) {} static void OnFree(const void *ptr, uptr size) {} }; diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_primary64.h b/lib/tsan/sanitizer_common/sanitizer_allocator_primary64.h @@ -185,9 +185,10 @@ class SizeClassAllocator64 { // recoverable. if (UNLIKELY(!EnsureFreeArraySpace(region, region_beg, new_num_freed_chunks))) { - Report("FATAL: Internal error: %s's allocator exhausted the free list " - "space for size class %zd (%zd bytes).\n", SanitizerToolName, - class_id, ClassIdToSize(class_id)); + Report( + "FATAL: Internal error: %s's allocator exhausted the free list " + "space for size class %zu (%zu bytes).\n", + SanitizerToolName, class_id, ClassIdToSize(class_id)); Die(); } for (uptr i = 0; i < n_chunks; i++) @@ -763,8 +764,9 @@ class SizeClassAllocator64 { if (!region->exhausted) { region->exhausted = true; Printf("%s: Out of memory. ", SanitizerToolName); - Printf("The process has exhausted %zuMB for size class %zu.\n", - kRegionSize >> 20, ClassIdToSize(class_id)); + Printf( + "The process has exhausted %zu MB for size class %zu (%zu bytes).\n", + kRegionSize >> 20, class_id, ClassIdToSize(class_id)); } return true; } diff --git a/lib/tsan/sanitizer_common/sanitizer_common.h b/lib/tsan/sanitizer_common/sanitizer_common.h @@ -83,8 +83,8 @@ int TgKill(pid_t pid, tid_t tid, int sig); uptr GetThreadSelf(); void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, uptr *stack_bottom); -void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, - uptr *tls_addr, uptr *tls_size); +void GetThreadStackAndTls(bool main, uptr *stk_begin, uptr *stk_end, + uptr *tls_begin, uptr *tls_end); // Memory management void *MmapOrDie(uptr size, const char *mem_type, bool raw_report = false); @@ -239,13 +239,15 @@ void RemoveANSIEscapeSequencesFromString(char *buffer); void Printf(const char *format, ...) FORMAT(1, 2); void Report(const char *format, ...) FORMAT(1, 2); void SetPrintfAndReportCallback(void (*callback)(const char *)); -#define VReport(level, ...) \ - do { \ - if ((uptr)Verbosity() >= (level)) Report(__VA_ARGS__); \ +#define VReport(level, ...) \ + do { \ + if (UNLIKELY((uptr)Verbosity() >= (level))) \ + Report(__VA_ARGS__); \ } while (0) -#define VPrintf(level, ...) \ - do { \ - if ((uptr)Verbosity() >= (level)) Printf(__VA_ARGS__); \ +#define VPrintf(level, ...) \ + do { \ + if (UNLIKELY((uptr)Verbosity() >= (level))) \ + Printf(__VA_ARGS__); \ } while (0) // Lock sanitizer error reporting and protects against nested errors. @@ -266,7 +268,15 @@ class ScopedErrorReportLock { extern uptr stoptheworld_tracer_pid; extern uptr stoptheworld_tracer_ppid; +// Returns true if the entire range can be read. bool IsAccessibleMemoryRange(uptr beg, uptr size); +// Attempts to copy `n` bytes from memory range starting at `src` to `dest`. +// Returns true if the entire range can be read. Returns `false` if any part of +// the source range cannot be read, in which case the contents of `dest` are +// undefined. +bool TryMemCpy(void *dest, const void *src, uptr n); +// Copies accessible memory, and zero fill inaccessible. +void MemCpyAccessible(void *dest, const void *src, uptr n); // Error report formatting. const char *StripPathPrefix(const char *filepath, @@ -917,7 +927,6 @@ typedef void (*RangeIteratorCallback)(uptr begin, uptr end, void *arg); enum AndroidApiLevel { ANDROID_NOT_ANDROID = 0, - ANDROID_KITKAT = 19, ANDROID_LOLLIPOP_MR1 = 22, ANDROID_POST_LOLLIPOP = 23 }; diff --git a/lib/tsan/sanitizer_common/sanitizer_common_interceptors.inc b/lib/tsan/sanitizer_common/sanitizer_common_interceptors.inc @@ -41,6 +41,7 @@ #include "sanitizer_errno.h" #include "sanitizer_placement_new.h" #include "sanitizer_platform_interceptors.h" +#include "sanitizer_platform_limits_posix.h" #include "sanitizer_symbolizer.h" #include "sanitizer_tls_get_addr.h" @@ -127,6 +128,39 @@ extern const short *_toupper_tab_; extern const short *_tolower_tab_; #endif +#if SANITIZER_LINUX && SANITIZER_SPARC32 +// On 32-bit Linux/sparc64, double and long double are identical and glibc +// uses a __nldbl_ (no long double) prefix for various stdio functions. +# define __isoc23_fscanf __nldbl___isoc23_fscanf +# define __isoc23_scanf __nldbl___isoc23_scanf +# define __isoc23_sscanf __nldbl___isoc23_sscanf +# define __isoc23_vfscanf __nldbl___isoc23_vfscanf +# define __isoc23_vscanf __nldbl___isoc23_vscanf +# define __isoc23_vsscanf __nldbl___isoc23_vsscanf +# define __isoc99_fscanf __nldbl___isoc99_fscanf +# define __isoc99_scanf __nldbl___isoc99_scanf +# define __isoc99_sscanf __nldbl___isoc99_sscanf +# define __isoc99_vfscanf __nldbl___isoc99_vfscanf +# define __isoc99_vscanf __nldbl___isoc99_vscanf +# define __isoc99_vsscanf __nldbl___isoc99_vsscanf +# define asprintf __nldbl_asprintf +# define fprintf __nldbl_fprintf +# define fscanf __nldbl_fscanf +# define printf __nldbl_printf +# define scanf __nldbl_scanf +# define snprintf __nldbl_snprintf +# define sprintf __nldbl_sprintf +# define sscanf __nldbl_sscanf +# define vasprintf __nldbl_vasprintf +# define vfprintf __nldbl_vfprintf +# define vfscanf __nldbl_vfscanf +# define vprintf __nldbl_vprintf +# define vscanf __nldbl_vscanf +# define vsnprintf __nldbl_vsnprintf +# define vsprintf __nldbl_vsprintf +# define vsscanf __nldbl_vsscanf +#endif + #if SANITIZER_MUSL && \ (defined(__i386__) || defined(__arm__) || SANITIZER_MIPS32 || SANITIZER_PPC32) // musl 1.2.0 on existing 32-bit architectures uses new symbol names for the @@ -313,7 +347,7 @@ extern const short *_tolower_tab_; uptr copy_length = internal_strnlen(s, size); \ char *new_mem = (char *)WRAP(malloc)(copy_length + 1); \ if (common_flags()->intercept_strndup) { \ - COMMON_INTERCEPTOR_READ_STRING(ctx, s, Min(size, copy_length + 1)); \ + COMMON_INTERCEPTOR_READ_STRING(ctx, s, Min<uptr>(size, copy_length + 1)); \ } \ if (new_mem) { \ COMMON_INTERCEPTOR_COPY_STRING(ctx, new_mem, s, copy_length); \ @@ -411,7 +445,7 @@ INTERCEPTOR(SIZE_T, strnlen, const char *s, SIZE_T maxlen) { #endif #if SANITIZER_INTERCEPT_STRNDUP -INTERCEPTOR(char*, strndup, const char *s, uptr size) { +INTERCEPTOR(char*, strndup, const char *s, usize size) { void *ctx; COMMON_INTERCEPTOR_STRNDUP_IMPL(ctx, s, size); } @@ -421,7 +455,7 @@ INTERCEPTOR(char*, strndup, const char *s, uptr size) { #endif // SANITIZER_INTERCEPT_STRNDUP #if SANITIZER_INTERCEPT___STRNDUP -INTERCEPTOR(char*, __strndup, const char *s, uptr size) { +INTERCEPTOR(char*, __strndup, const char *s, usize size) { void *ctx; COMMON_INTERCEPTOR_STRNDUP_IMPL(ctx, s, size); } @@ -477,23 +511,23 @@ INTERCEPTOR(int, strcmp, const char *s1, const char *s2) { } DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncmp, uptr called_pc, - const char *s1, const char *s2, uptr n, + const char *s1, const char *s2, usize n, int result) -INTERCEPTOR(int, strncmp, const char *s1, const char *s2, uptr size) { +INTERCEPTOR(int, strncmp, const char *s1, const char *s2, usize size) { if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) return internal_strncmp(s1, s2, size); void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, strncmp, s1, s2, size); unsigned char c1 = 0, c2 = 0; - uptr i; + usize i; for (i = 0; i < size; i++) { c1 = (unsigned char)s1[i]; c2 = (unsigned char)s2[i]; if (c1 != c2 || c1 == '\0') break; } - uptr i1 = i; - uptr i2 = i; + usize i1 = i; + usize i2 = i; if (common_flags()->strict_string_checks) { for (; i1 < size && s1[i1]; i1++) {} for (; i2 < size && s2[i2]; i2++) {} @@ -542,21 +576,21 @@ INTERCEPTOR(int, strcasecmp, const char *s1, const char *s2) { } DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncasecmp, uptr called_pc, - const char *s1, const char *s2, uptr size, + const char *s1, const char *s2, usize size, int result) INTERCEPTOR(int, strncasecmp, const char *s1, const char *s2, SIZE_T size) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, strncasecmp, s1, s2, size); unsigned char c1 = 0, c2 = 0; - uptr i; + usize i; for (i = 0; i < size; i++) { c1 = (unsigned char)s1[i]; c2 = (unsigned char)s2[i]; if (CharCaseCmp(c1, c2) != 0 || c1 == '\0') break; } - uptr i1 = i; - uptr i2 = i; + usize i1 = i; + usize i2 = i; if (common_flags()->strict_string_checks) { for (; i1 < size && s1[i1]; i1++) {} for (; i2 < size && s2[i2]; i2++) {} @@ -799,13 +833,13 @@ INTERCEPTOR(char *, strpbrk, const char *s1, const char *s2) { #if SANITIZER_INTERCEPT_MEMCMP DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc, - const void *s1, const void *s2, uptr n, + const void *s1, const void *s2, usize n, int result) // Common code for `memcmp` and `bcmp`. int MemcmpInterceptorCommon(void *ctx, - int (*real_fn)(const void *, const void *, uptr), - const void *a1, const void *a2, uptr size) { + int (*real_fn)(const void *, const void *, usize), + const void *a1, const void *a2, usize size) { if (common_flags()->intercept_memcmp) { if (common_flags()->strict_memcmp) { // Check the entire regions even if the first bytes of the buffers are @@ -817,7 +851,7 @@ int MemcmpInterceptorCommon(void *ctx, unsigned char c1 = 0, c2 = 0; const unsigned char *s1 = (const unsigned char*)a1; const unsigned char *s2 = (const unsigned char*)a2; - uptr i; + usize i; for (i = 0; i < size; i++) { c1 = s1[i]; c2 = s2[i]; @@ -837,7 +871,7 @@ int MemcmpInterceptorCommon(void *ctx, return result; } -INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) { +INTERCEPTOR(int, memcmp, const void *a1, const void *a2, usize size) { if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) return internal_memcmp(a1, a2, size); void *ctx; @@ -851,7 +885,7 @@ INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) { #endif #if SANITIZER_INTERCEPT_BCMP -INTERCEPTOR(int, bcmp, const void *a1, const void *a2, uptr size) { +INTERCEPTOR(int, bcmp, const void *a1, const void *a2, usize size) { if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) return internal_memcmp(a1, a2, size); void *ctx; @@ -1104,7 +1138,7 @@ INTERCEPTOR(SSIZE_T, write, int fd, void *ptr, SIZE_T count) { #endif #if SANITIZER_INTERCEPT_FWRITE -INTERCEPTOR(SIZE_T, fwrite, const void *p, uptr size, uptr nmemb, void *file) { +INTERCEPTOR(SIZE_T, fwrite, const void *p, usize size, usize nmemb, void *file) { // libc file streams can call user-supplied functions, see fopencookie. void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, fwrite, p, size, nmemb, file); @@ -1255,6 +1289,12 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3, static const int PR_SET_VMA = 0x53564d41; static const int PR_SCHED_CORE = 62; static const int PR_SCHED_CORE_GET = 0; + static const int PR_GET_PDEATHSIG = 2; + +# if !SANITIZER_ANDROID + static const int PR_SET_SECCOMP = 22; + static const int SECCOMP_MODE_FILTER = 2; +# endif if (option == PR_SET_VMA && arg2 == 0UL) { char *name = (char *)arg5; COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1); @@ -1270,7 +1310,14 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3, COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1); } else if (res != -1 && option == PR_SCHED_CORE && arg2 == PR_SCHED_CORE_GET) { - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64*)(arg5), sizeof(u64)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg5), sizeof(u64)); + } else if (res != -1 && option == PR_GET_PDEATHSIG) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg2), sizeof(int)); +# if SANITIZER_GLIBC + } else if (res != -1 && option == PR_SET_SECCOMP && + arg2 == SECCOMP_MODE_FILTER) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg3), struct_sock_fprog_sz); +# endif } return res; } @@ -2242,6 +2289,61 @@ INTERCEPTOR(int, pthread_getcpuclockid, uptr thread, #define INIT_CLOCK_GETCPUCLOCKID #endif +#if SANITIZER_INTERCEPT_TIMER_CREATE +INTERCEPTOR(int, timer_create, __sanitizer_clockid_t clockid, void *sevp, + __sanitizer_timer_t *timer) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, timer_create, clockid, sevp, timer); + int res = REAL(timer_create)(clockid, sevp, timer); + if (!res && timer) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, timer, sizeof *timer); + } + return res; +} + +INTERCEPTOR(int, timer_delete, __sanitizer_timer_t timer) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, timer_delete, timer); + int res = REAL(timer_delete)(timer); + return res; +} + +INTERCEPTOR(int, timer_gettime, __sanitizer_timer_t timer, + struct __sanitizer_itimerspec *curr_value) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, timer_gettime, timer, curr_value); + int res = REAL(timer_gettime)(timer, curr_value); + if (!res && curr_value) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, curr_value, sizeof *curr_value); + } + return res; +} + +INTERCEPTOR(int, timer_settime, __sanitizer_timer_t timer, int flags, + const struct __sanitizer_itimerspec *new_value, + struct __sanitizer_itimerspec *old_value) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, timer_settime, timer, flags, new_value, + old_value); + int res = REAL(timer_settime)(timer, flags, new_value, old_value); + if (!res) { + if (new_value) + COMMON_INTERCEPTOR_READ_RANGE(ctx, new_value, sizeof *new_value); + if (old_value) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, old_value, sizeof *old_value); + } + return res; +} + +# define INIT_TIMER_CREATE \ + COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(timer_create, "GLIBC_2.3.3"); \ + COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(timer_delete, "GLIBC_2.3.3"); \ + COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(timer_gettime, "GLIBC_2.3.3"); \ + COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(timer_settime, "GLIBC_2.3.3"); +#else +# define INIT_TIMER_CREATE +#endif + #if SANITIZER_INTERCEPT_GETITIMER INTERCEPTOR(int, getitimer, int which, void *curr_value) { void *ctx; @@ -2287,6 +2389,25 @@ INTERCEPTOR(int, setitimer, int which, const void *new_value, void *old_value) { #define INIT_GETITIMER #endif +#if SANITIZER_INTERCEPT_TIMESPEC_GET +INTERCEPTOR(int, timespec_get, struct __sanitizer_timespec *ts, int base) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, timespec_get, ts, base); + // We don't yet know if ts is addressable, so we use our own scratch buffer + struct __sanitizer_timespec ts_local; + int res = REAL(timespec_get)(&ts_local, base); + if (res) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ts, + sizeof(struct __sanitizer_timespec)); + internal_memcpy(ts, &ts_local, sizeof(struct __sanitizer_timespec)); + } + return res; +} +# define INIT_TIMESPEC_GET COMMON_INTERCEPT_FUNCTION(timespec_get); +#else +# define INIT_TIMESPEC_GET +#endif + #if SANITIZER_INTERCEPT_GLOB static void unpoison_glob_t(void *ctx, __sanitizer_glob_t *pglob) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pglob, sizeof(*pglob)); @@ -3427,23 +3548,27 @@ INTERCEPTOR(uptr, ptrace, int request, int pid, void *addr, void *data) { COMMON_INTERCEPTOR_ENTER(ctx, ptrace, request, pid, addr, data); __sanitizer_iovec local_iovec; - if (data) { + void *data_arg = ptrace_data_arg(request, addr, data); + if (data_arg) { if (request == ptrace_setregs) { - COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_regs_struct_sz); + COMMON_INTERCEPTOR_READ_RANGE(ctx, data_arg, struct_user_regs_struct_sz); } else if (request == ptrace_setfpregs) { - COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_fpregs_struct_sz); + COMMON_INTERCEPTOR_READ_RANGE(ctx, data_arg, + struct_user_fpregs_struct_sz); } else if (request == ptrace_setfpxregs) { - COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_fpxregs_struct_sz); + COMMON_INTERCEPTOR_READ_RANGE(ctx, data_arg, + struct_user_fpxregs_struct_sz); } else if (request == ptrace_setvfpregs) { - COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_vfpregs_struct_sz); + COMMON_INTERCEPTOR_READ_RANGE(ctx, data_arg, + struct_user_vfpregs_struct_sz); } else if (request == ptrace_setsiginfo) { - COMMON_INTERCEPTOR_READ_RANGE(ctx, data, siginfo_t_sz); + COMMON_INTERCEPTOR_READ_RANGE(ctx, data_arg, siginfo_t_sz); - // Some kernel might zero the iovec::iov_base in case of invalid - // write access. In this case copy the invalid address for further - // inspection. + // Some kernel might zero the iovec::iov_base in case of invalid + // write access. In this case copy the invalid address for further + // inspection. } else if (request == ptrace_setregset || request == ptrace_getregset) { - __sanitizer_iovec *iovec = (__sanitizer_iovec*)data; + __sanitizer_iovec *iovec = (__sanitizer_iovec *)data_arg; COMMON_INTERCEPTOR_READ_RANGE(ctx, iovec, sizeof(*iovec)); local_iovec = *iovec; if (request == ptrace_setregset) @@ -3456,23 +3581,26 @@ INTERCEPTOR(uptr, ptrace, int request, int pid, void *addr, void *data) { // https://github.com/google/sanitizers/issues/321. uptr res = REAL(ptrace)(request, pid, addr, data); - if (!res && data) { + if (!res && data_arg) { // Note that PEEK* requests assign different meaning to the return value. // This function does not handle them (nor does it need to). if (request == ptrace_getregs) { - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_regs_struct_sz); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg, struct_user_regs_struct_sz); } else if (request == ptrace_getfpregs) { - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_fpregs_struct_sz); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg, + struct_user_fpregs_struct_sz); } else if (request == ptrace_getfpxregs) { - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_fpxregs_struct_sz); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg, + struct_user_fpxregs_struct_sz); } else if (request == ptrace_getvfpregs) { - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_vfpregs_struct_sz); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg, + struct_user_vfpregs_struct_sz); } else if (request == ptrace_getsiginfo) { - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, siginfo_t_sz); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg, siginfo_t_sz); } else if (request == ptrace_geteventmsg) { - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, sizeof(unsigned long)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg, sizeof(unsigned long)); } else if (request == ptrace_getregset) { - __sanitizer_iovec *iovec = (__sanitizer_iovec*)data; + __sanitizer_iovec *iovec = (__sanitizer_iovec *)data_arg; COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iovec, sizeof(*iovec)); COMMON_INTERCEPTOR_WRITE_RANGE(ctx, local_iovec.iov_base, local_iovec.iov_len); @@ -6425,12 +6553,12 @@ static void MlockIsUnsupported() { SanitizerToolName); } -INTERCEPTOR(int, mlock, const void *addr, uptr len) { +INTERCEPTOR(int, mlock, const void *addr, usize len) { MlockIsUnsupported(); return 0; } -INTERCEPTOR(int, munlock, const void *addr, uptr len) { +INTERCEPTOR(int, munlock, const void *addr, usize len) { MlockIsUnsupported(); return 0; } @@ -8823,83 +8951,6 @@ INTERCEPTOR(char *, RMD160Data, u8 *data, SIZE_T len, char *buf) { #define INIT_RMD160 #endif -#if SANITIZER_INTERCEPT_MD5 -INTERCEPTOR(void, MD5Init, void *context) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, MD5Init, context); - REAL(MD5Init)(context); - if (context) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD5_CTX_sz); -} - -INTERCEPTOR(void, MD5Update, void *context, const unsigned char *data, - unsigned int len) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, MD5Update, context, data, len); - if (data && len > 0) - COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); - if (context) - COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz); - REAL(MD5Update)(context, data, len); - if (context) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD5_CTX_sz); -} - -INTERCEPTOR(void, MD5Final, unsigned char digest[16], void *context) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, MD5Final, digest, context); - if (context) - COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz); - REAL(MD5Final)(digest, context); - if (digest) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(unsigned char) * 16); -} - -INTERCEPTOR(char *, MD5End, void *context, char *buf) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, MD5End, context, buf); - if (context) - COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz); - char *ret = REAL(MD5End)(context, buf); - if (ret) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length); - return ret; -} - -INTERCEPTOR(char *, MD5File, const char *filename, char *buf) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, MD5File, filename, buf); - if (filename) - COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1); - char *ret = REAL(MD5File)(filename, buf); - if (ret) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length); - return ret; -} - -INTERCEPTOR(char *, MD5Data, const unsigned char *data, unsigned int len, - char *buf) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, MD5Data, data, len, buf); - if (data && len > 0) - COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); - char *ret = REAL(MD5Data)(data, len, buf); - if (ret) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length); - return ret; -} - -#define INIT_MD5 \ - COMMON_INTERCEPT_FUNCTION(MD5Init); \ - COMMON_INTERCEPT_FUNCTION(MD5Update); \ - COMMON_INTERCEPT_FUNCTION(MD5Final); \ - COMMON_INTERCEPT_FUNCTION(MD5End); \ - COMMON_INTERCEPT_FUNCTION(MD5File); \ - COMMON_INTERCEPT_FUNCTION(MD5Data) -#else -#define INIT_MD5 -#endif - #if SANITIZER_INTERCEPT_FSEEK INTERCEPTOR(int, fseek, __sanitizer_FILE *stream, long int offset, int whence) { void *ctx; @@ -9030,107 +9081,6 @@ INTERCEPTOR(char *, MD2Data, const unsigned char *data, unsigned int len, #define INIT_MD2 #endif -#if SANITIZER_INTERCEPT_SHA2 -#define SHA2_INTERCEPTORS(LEN, SHA2_STATE_T) \ - INTERCEPTOR(void, SHA##LEN##_Init, void *context) { \ - void *ctx; \ - COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Init, context); \ - REAL(SHA##LEN##_Init)(context); \ - if (context) \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA##LEN##_CTX_sz); \ - } \ - INTERCEPTOR(void, SHA##LEN##_Update, void *context, \ - const u8 *data, SIZE_T len) { \ - void *ctx; \ - COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Update, context, data, len); \ - if (data && len > 0) \ - COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); \ - if (context) \ - COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \ - REAL(SHA##LEN##_Update)(context, data, len); \ - if (context) \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA##LEN##_CTX_sz); \ - } \ - INTERCEPTOR(void, SHA##LEN##_Final, u8 digest[LEN/8], \ - void *context) { \ - void *ctx; \ - CHECK_EQ(SHA##LEN##_digest_length, LEN/8); \ - COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Final, digest, context); \ - if (context) \ - COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \ - REAL(SHA##LEN##_Final)(digest, context); \ - if (digest) \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, \ - sizeof(digest[0]) * \ - SHA##LEN##_digest_length); \ - } \ - INTERCEPTOR(char *, SHA##LEN##_End, void *context, char *buf) { \ - void *ctx; \ - COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_End, context, buf); \ - if (context) \ - COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \ - char *ret = REAL(SHA##LEN##_End)(context, buf); \ - if (ret) \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \ - return ret; \ - } \ - INTERCEPTOR(char *, SHA##LEN##_File, const char *filename, char *buf) { \ - void *ctx; \ - COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_File, filename, buf); \ - if (filename) \ - COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);\ - char *ret = REAL(SHA##LEN##_File)(filename, buf); \ - if (ret) \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \ - return ret; \ - } \ - INTERCEPTOR(char *, SHA##LEN##_FileChunk, const char *filename, char *buf, \ - OFF_T offset, OFF_T length) { \ - void *ctx; \ - COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_FileChunk, filename, buf, offset, \ - length); \ - if (filename) \ - COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);\ - char *ret = REAL(SHA##LEN##_FileChunk)(filename, buf, offset, length); \ - if (ret) \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \ - return ret; \ - } \ - INTERCEPTOR(char *, SHA##LEN##_Data, u8 *data, SIZE_T len, char *buf) { \ - void *ctx; \ - COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Data, data, len, buf); \ - if (data && len > 0) \ - COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); \ - char *ret = REAL(SHA##LEN##_Data)(data, len, buf); \ - if (ret) \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \ - return ret; \ - } - -SHA2_INTERCEPTORS(224, u32) -SHA2_INTERCEPTORS(256, u32) -SHA2_INTERCEPTORS(384, u64) -SHA2_INTERCEPTORS(512, u64) - -#define INIT_SHA2_INTECEPTORS(LEN) \ - COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Init); \ - COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Update); \ - COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Final); \ - COMMON_INTERCEPT_FUNCTION(SHA##LEN##_End); \ - COMMON_INTERCEPT_FUNCTION(SHA##LEN##_File); \ - COMMON_INTERCEPT_FUNCTION(SHA##LEN##_FileChunk); \ - COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Data) - -#define INIT_SHA2 \ - INIT_SHA2_INTECEPTORS(224); \ - INIT_SHA2_INTECEPTORS(256); \ - INIT_SHA2_INTECEPTORS(384); \ - INIT_SHA2_INTECEPTORS(512) -#undef SHA2_INTERCEPTORS -#else -#define INIT_SHA2 -#endif - #if SANITIZER_INTERCEPT_VIS INTERCEPTOR(char *, vis, char *dst, int c, int flag, int nextc) { void *ctx; @@ -9980,7 +9930,7 @@ INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getrandom, buf, buflen, flags); // If GRND_NONBLOCK is set in the flags, it is non blocking. - static const int grnd_nonblock = 1; + static const int grnd_nonblock = 1; SSIZE_T n; if ((flags & grnd_nonblock)) n = REAL(getrandom)(buf, buflen, flags); @@ -10296,6 +10246,23 @@ INTERCEPTOR(SSIZE_T, pwritev2, int fd, __sanitizer_iovec *iov, int iovcnt, #define INIT_PWRITEV2 #endif +#if SANITIZER_INTERCEPT_FREADLINK +INTERCEPTOR(SSIZE_T, freadlink, int fd, char *buf, SIZE_T bufsiz) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, freadlink, fd, buf, bufsiz); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + SSIZE_T res = REAL(freadlink)(fd, buf, bufsiz); + if (res > 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res); + if (res >= 0 && fd > 0) + COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); + return res; +} +# define INIT_FREADLINK COMMON_INTERCEPT_FUNCTION(freadlink) +#else +# define INIT_FREADLINK +#endif + #include "sanitizer_common_interceptors_netbsd_compat.inc" namespace __sanitizer { @@ -10373,8 +10340,10 @@ static void InitializeCommonInterceptors() { INIT_SETPWENT; INIT_CLOCK_GETTIME; INIT_CLOCK_GETCPUCLOCKID; + INIT_TIMER_CREATE; INIT_GETITIMER; INIT_TIME; + INIT_TIMESPEC_GET; INIT_GLOB; INIT_GLOB64; INIT___B64_TO; @@ -10588,10 +10557,8 @@ static void InitializeCommonInterceptors() { INIT_SHA1; INIT_MD4; INIT_RMD160; - INIT_MD5; INIT_FSEEK; INIT_MD2; - INIT_SHA2; INIT_VIS; INIT_CDB; INIT_GETFSENT; @@ -10617,6 +10584,7 @@ static void InitializeCommonInterceptors() { INIT_CPUSET_GETAFFINITY; INIT_PREADV2; INIT_PWRITEV2; + INIT_FREADLINK; INIT___PRINTF_CHK; } diff --git a/lib/tsan/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc b/lib/tsan/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc @@ -82,7 +82,7 @@ #endif #if SANITIZER_INTERCEPT_MEMSET -INTERCEPTOR(void *, memset, void *dst, int v, uptr size) { +INTERCEPTOR(void *, memset, void *dst, int v, usize size) { void *ctx; COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size); } @@ -93,7 +93,7 @@ INTERCEPTOR(void *, memset, void *dst, int v, uptr size) { #endif #if SANITIZER_INTERCEPT_MEMMOVE -INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) { +INTERCEPTOR(void *, memmove, void *dst, const void *src, usize size) { void *ctx; COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size); } @@ -104,7 +104,7 @@ INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) { #endif #if SANITIZER_INTERCEPT_MEMCPY -INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) { +INTERCEPTOR(void *, memcpy, void *dst, const void *src, usize size) { // On OS X, calling internal_memcpy here will cause memory corruptions, // because memcpy and memmove are actually aliases of the same // implementation. We need to use internal_memmove here. @@ -133,63 +133,63 @@ INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) { #endif #if SANITIZER_INTERCEPT_AEABI_MEM -INTERCEPTOR(void *, __aeabi_memmove, void *to, const void *from, uptr size) { +INTERCEPTOR(void *, __aeabi_memmove, void *to, const void *from, usize size) { void *ctx; COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size); } -INTERCEPTOR(void *, __aeabi_memmove4, void *to, const void *from, uptr size) { +INTERCEPTOR(void *, __aeabi_memmove4, void *to, const void *from, usize size) { void *ctx; COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size); } -INTERCEPTOR(void *, __aeabi_memmove8, void *to, const void *from, uptr size) { +INTERCEPTOR(void *, __aeabi_memmove8, void *to, const void *from, usize size) { void *ctx; COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size); } -INTERCEPTOR(void *, __aeabi_memcpy, void *to, const void *from, uptr size) { +INTERCEPTOR(void *, __aeabi_memcpy, void *to, const void *from, usize size) { void *ctx; COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size); } -INTERCEPTOR(void *, __aeabi_memcpy4, void *to, const void *from, uptr size) { +INTERCEPTOR(void *, __aeabi_memcpy4, void *to, const void *from, usize size) { void *ctx; COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size); } -INTERCEPTOR(void *, __aeabi_memcpy8, void *to, const void *from, uptr size) { +INTERCEPTOR(void *, __aeabi_memcpy8, void *to, const void *from, usize size) { void *ctx; COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size); } // Note the argument order. -INTERCEPTOR(void *, __aeabi_memset, void *block, uptr size, int c) { +INTERCEPTOR(void *, __aeabi_memset, void *block, usize size, int c) { void *ctx; COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size); } -INTERCEPTOR(void *, __aeabi_memset4, void *block, uptr size, int c) { +INTERCEPTOR(void *, __aeabi_memset4, void *block, usize size, int c) { void *ctx; COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size); } -INTERCEPTOR(void *, __aeabi_memset8, void *block, uptr size, int c) { +INTERCEPTOR(void *, __aeabi_memset8, void *block, usize size, int c) { void *ctx; COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size); } -INTERCEPTOR(void *, __aeabi_memclr, void *block, uptr size) { +INTERCEPTOR(void *, __aeabi_memclr, void *block, usize size) { void *ctx; COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); } -INTERCEPTOR(void *, __aeabi_memclr4, void *block, uptr size) { +INTERCEPTOR(void *, __aeabi_memclr4, void *block, usize size) { void *ctx; COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); } -INTERCEPTOR(void *, __aeabi_memclr8, void *block, uptr size) { +INTERCEPTOR(void *, __aeabi_memclr8, void *block, usize size) { void *ctx; COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); } @@ -212,7 +212,7 @@ INTERCEPTOR(void *, __aeabi_memclr8, void *block, uptr size) { #endif // SANITIZER_INTERCEPT_AEABI_MEM #if SANITIZER_INTERCEPT___BZERO -INTERCEPTOR(void *, __bzero, void *block, uptr size) { +INTERCEPTOR(void *, __bzero, void *block, usize size) { void *ctx; COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); } @@ -222,7 +222,7 @@ INTERCEPTOR(void *, __bzero, void *block, uptr size) { #endif // SANITIZER_INTERCEPT___BZERO #if SANITIZER_INTERCEPT_BZERO -INTERCEPTOR(void *, bzero, void *block, uptr size) { +INTERCEPTOR(void *, bzero, void *block, usize size) { void *ctx; COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); } diff --git a/lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S b/lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S @@ -0,0 +1,48 @@ +#if defined(__aarch64__) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" +#include "builtins/assembly.h" + +ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA) + +.comm _ZN14__interception10real_vforkE,8,8 +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + // Save x30 in the off-stack spill area. + hint #25 // paciasp + stp xzr, x30, [sp, #-16]! + bl COMMON_INTERCEPTOR_SPILL_AREA + ldp xzr, x30, [sp], 16 + str x30, [x0] + + // Call real vfork. This may return twice. User code that runs between the first and the second return + // may clobber the stack frame of the interceptor; that's why it does not have a frame. + adrp x0, _ZN14__interception10real_vforkE + ldr x0, [x0, :lo12:_ZN14__interception10real_vforkE] + blr x0 + + stp x0, xzr, [sp, #-16]! + cmp x0, #0 + b.eq .L_exit + + // x0 != 0 => parent process. Clear stack shadow. + add x0, sp, #16 + bl COMMON_INTERCEPTOR_HANDLE_VFORK + +.L_exit: + // Restore x30. + bl COMMON_INTERCEPTOR_SPILL_AREA + ldr x30, [x0] + ldp x0, xzr, [sp], 16 + hint #29 // autiasp + + ret +ASM_SIZE(vfork) + +ASM_INTERCEPTOR_TRAMPOLINE(vfork) +ASM_TRAMPOLINE_ALIAS(vfork, vfork) + +GNU_PROPERTY_BTI_PAC + +#endif diff --git a/lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S b/lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S @@ -0,0 +1,49 @@ +#if defined(__arm__) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" + +ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA) + +.comm _ZN14__interception10real_vforkE,4,4 +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + // Save LR in the off-stack spill area. + push {r4, lr} + bl COMMON_INTERCEPTOR_SPILL_AREA + pop {r4, lr} + str lr, [r0] + + // Call real vfork. This may return twice. User code that runs between the first and the second return + // may clobber the stack frame of the interceptor; that's why it does not have a frame. + ldr r0, .LCPI0_0 +.LPC0_0: + ldr r0, [pc, r0] + mov lr, pc + bx r0 + + push {r0, r4} + cmp r0, #0 + beq .L_exit + + // r0 != 0 => parent process. Clear stack shadow. + add r0, sp, #8 + bl COMMON_INTERCEPTOR_HANDLE_VFORK + +.L_exit: + // Restore LR. + bl COMMON_INTERCEPTOR_SPILL_AREA + ldr lr, [r0] + pop {r0, r4} + + mov pc, lr + +.LCPI0_0: + .long _ZN14__interception10real_vforkE - (.LPC0_0+8) + +ASM_SIZE(vfork) + +ASM_INTERCEPTOR_TRAMPOLINE(vfork) +ASM_TRAMPOLINE_ALIAS(vfork, vfork) + +#endif diff --git a/lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S b/lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S @@ -0,0 +1,64 @@ +#if defined(__i386__) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" + +.comm _ZN14__interception10real_vforkE,4,4 +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + _CET_ENDBR + // Store return address in the spill area and tear down the stack frame. + sub $12, %esp + call COMMON_INTERCEPTOR_SPILL_AREA + mov 12(%esp), %ecx + mov %ecx, (%eax) + add $16, %esp + + call .L0$pb +.L0$pb: + pop %eax +.Ltmp0: + add $_GLOBAL_OFFSET_TABLE_+(.Ltmp0-.L0$pb), %eax + call *_ZN14__interception10real_vforkE@GOTOFF(%eax) + + // Restore the stack frame. + // 12(%esp) return address + // 8(%esp) spill %ebx + // 4(%esp) spill REAL(vfork) return value + // (%esp) call frame (arg0) for __*_handle_vfork + sub $16, %esp + mov %ebx, 8(%esp) + mov %eax, 4(%esp) + + // Form GOT address in %ebx. + call .L1$pb +.L1$pb: + pop %ebx +.Ltmp1: + add $_GLOBAL_OFFSET_TABLE_+(.Ltmp1-.L1$pb), %ebx + + // Restore original return address. + call COMMON_INTERCEPTOR_SPILL_AREA + mov (%eax), %ecx + mov %ecx, 12(%esp) + mov 4(%esp), %eax + + // Call handle_vfork in the parent process (%rax != 0). + test %eax, %eax + je .L_exit + + lea 16(%esp), %ecx + mov %ecx, (%esp) + call COMMON_INTERCEPTOR_HANDLE_VFORK@PLT + +.L_exit: + mov 4(%esp), %eax + mov 8(%esp), %ebx + add $12, %esp + ret +ASM_SIZE(vfork) + +ASM_INTERCEPTOR_TRAMPOLINE(vfork) +ASM_TRAMPOLINE_ALIAS(vfork, vfork) + +#endif diff --git a/lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S b/lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S @@ -0,0 +1,57 @@ +#if defined(__loongarch_lp64) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" + +ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA) +ASM_HIDDEN(_ZN14__interception10real_vforkE) + +.text +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + // Save ra in the off-stack spill area. + // allocate space on stack + addi.d $sp, $sp, -16 + // store $ra value + st.d $ra, $sp, 8 + bl COMMON_INTERCEPTOR_SPILL_AREA + // restore previous values from stack + ld.d $ra, $sp, 8 + // adjust stack + addi.d $sp, $sp, 16 + // store $ra by $a0 + st.d $ra, $a0, 0 + + // Call real vfork. This may return twice. User code that runs between the first and the second return + // may clobber the stack frame of the interceptor; that's why it does not have a frame. + la.local $a0, _ZN14__interception10real_vforkE + ld.d $a0, $a0, 0 + jirl $ra, $a0, 0 + + // adjust stack + addi.d $sp, $sp, -16 + // store $a0 by adjusted stack + st.d $a0, $sp, 8 + // jump to exit label if $a0 is 0 + beqz $a0, .L_exit + + // $a0 != 0 => parent process. Clear stack shadow. + // put old $sp to $a0 + addi.d $a0, $sp, 16 + bl %plt(COMMON_INTERCEPTOR_HANDLE_VFORK) + +.L_exit: + // Restore $ra + bl COMMON_INTERCEPTOR_SPILL_AREA + ld.d $ra, $a0, 0 + // load value by stack + ld.d $a0, $sp, 8 + // adjust stack + addi.d $sp, $sp, 16 + jr $ra +ASM_SIZE(vfork) + +ASM_INTERCEPTOR_TRAMPOLINE(vfork) +ASM_TRAMPOLINE_ALIAS(vfork, vfork) + +#endif diff --git a/lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_riscv64.inc.S b/lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_riscv64.inc.S @@ -0,0 +1,56 @@ +#if (defined(__riscv) && (__riscv_xlen == 64)) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" + +ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA) + +.comm _ZN14__interception10real_vforkE,8,8 +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + // Save ra in the off-stack spill area. + // allocate space on stack + addi sp, sp, -16 + // store ra value + sd ra, 8(sp) + call COMMON_INTERCEPTOR_SPILL_AREA + // restore previous values from stack + ld ra, 8(sp) + // adjust stack + addi sp, sp, 16 + // store ra by x10 + sd ra, 0(x10) + + // Call real vfork. This may return twice. User code that runs between the first and the second return + // may clobber the stack frame of the interceptor; that's why it does not have a frame. + la x10, _ZN14__interception10real_vforkE + ld x10, 0(x10) + jalr x10 + + // adjust stack + addi sp, sp, -16 + // store x10 by adjusted stack + sd x10, 8(sp) + // jump to exit label if x10 is 0 + beqz x10, .L_exit + + // x0 != 0 => parent process. Clear stack shadow. + // put old sp to x10 + addi x10, sp, 16 + call COMMON_INTERCEPTOR_HANDLE_VFORK + +.L_exit: + // Restore ra + call COMMON_INTERCEPTOR_SPILL_AREA + ld ra, 0(x10) + // load value by stack + ld x10, 8(sp) + // adjust stack + addi sp, sp, 16 + ret +ASM_SIZE(vfork) + +ASM_INTERCEPTOR_TRAMPOLINE(vfork) +ASM_TRAMPOLINE_ALIAS(vfork, vfork) + +#endif diff --git a/lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S b/lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S @@ -0,0 +1,42 @@ +#if defined(__x86_64__) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" + +.comm _ZN14__interception10real_vforkE,8,8 +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + _CET_ENDBR + // Store return address in the spill area and tear down the stack frame. + push %rcx + call COMMON_INTERCEPTOR_SPILL_AREA + pop %rcx + pop %rdi + mov %rdi, (%rax) + + call *_ZN14__interception10real_vforkE(%rip) + + // Restore return address from the spill area. + push %rcx + push %rax + call COMMON_INTERCEPTOR_SPILL_AREA + mov (%rax), %rdx + mov %rdx, 8(%rsp) + mov (%rsp), %rax + + // Call handle_vfork in the parent process (%rax != 0). + test %rax, %rax + je .L_exit + + lea 16(%rsp), %rdi + call COMMON_INTERCEPTOR_HANDLE_VFORK@PLT + +.L_exit: + pop %rax + ret +ASM_SIZE(ASM_WRAPPER_NAME(vfork)) + +ASM_INTERCEPTOR_TRAMPOLINE(vfork) +ASM_TRAMPOLINE_ALIAS(vfork, vfork) + +#endif diff --git a/lib/tsan/sanitizer_common/sanitizer_common_interface.inc b/lib/tsan/sanitizer_common/sanitizer_common_interface.inc @@ -10,6 +10,7 @@ INTERFACE_FUNCTION(__sanitizer_acquire_crash_state) INTERFACE_FUNCTION(__sanitizer_annotate_contiguous_container) INTERFACE_FUNCTION(__sanitizer_annotate_double_ended_contiguous_container) +INTERFACE_FUNCTION(__sanitizer_copy_contiguous_container_annotations) INTERFACE_FUNCTION(__sanitizer_contiguous_container_find_bad_address) INTERFACE_FUNCTION( __sanitizer_double_ended_contiguous_container_find_bad_address) @@ -22,6 +23,7 @@ INTERFACE_FUNCTION(__sanitizer_verify_double_ended_contiguous_container) INTERFACE_WEAK_FUNCTION(__sanitizer_on_print) INTERFACE_WEAK_FUNCTION(__sanitizer_report_error_summary) INTERFACE_WEAK_FUNCTION(__sanitizer_sandbox_on_notify) +INTERFACE_WEAK_FUNCTION(__sanitizer_get_dtls_size) // Sanitizer weak hooks INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_memcmp) INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_strcmp) @@ -51,3 +53,9 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_ignore_free_hook) INTERFACE_FUNCTION(__sanitizer_internal_memcpy) INTERFACE_FUNCTION(__sanitizer_internal_memmove) INTERFACE_FUNCTION(__sanitizer_internal_memset) + +#if SANITIZER_WINDOWS +INTERFACE_FUNCTION(__sanitizer_override_function) +INTERFACE_FUNCTION(__sanitizer_override_function_by_addr) +INTERFACE_FUNCTION(__sanitizer_register_weak_function) +#endif diff --git a/lib/tsan/sanitizer_common/sanitizer_common_libcdep.cpp b/lib/tsan/sanitizer_common/sanitizer_common_libcdep.cpp @@ -171,7 +171,7 @@ void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name, "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. " "Perhaps you're using ulimit -v or ulimit -d\n", size); - Abort(); + Die(); } if (madvise_shadow && common_flags()->use_madv_dontdump) DontDumpShadowMemory(beg, size); @@ -219,6 +219,32 @@ static void StopStackDepotBackgroundThread() { static void StopStackDepotBackgroundThread() {} #endif +void MemCpyAccessible(void *dest, const void *src, uptr n) { + if (TryMemCpy(dest, src, n)) + return; + + const uptr page_size = GetPageSize(); + uptr b = reinterpret_cast<uptr>(src); + uptr b_up = RoundUpTo(b, page_size); + + uptr e = reinterpret_cast<uptr>(src) + n; + uptr e_down = RoundDownTo(e, page_size); + + auto copy_or_zero = [dest, src](uptr beg, uptr end) { + const uptr udest = reinterpret_cast<uptr>(dest); + const uptr usrc = reinterpret_cast<uptr>(src); + void *d = reinterpret_cast<void *>(udest + (beg - usrc)); + const uptr size = end - beg; + if (!TryMemCpy(d, reinterpret_cast<void *>(beg), size)) + internal_memset(d, 0, size); + }; + + copy_or_zero(b, b_up); + for (uptr p = b_up; p < e_down; p += page_size) + copy_or_zero(p, p + page_size); + copy_or_zero(e_down, e); +} + } // namespace __sanitizer SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify, diff --git a/lib/tsan/sanitizer_common/sanitizer_common_nolibc.cpp b/lib/tsan/sanitizer_common/sanitizer_common_nolibc.cpp @@ -20,13 +20,14 @@ namespace __sanitizer { // The Windows implementations of these functions use the win32 API directly, // bypassing libc. #if !SANITIZER_WINDOWS -#if SANITIZER_LINUX +# if SANITIZER_LINUX void LogMessageOnPrintf(const char *str) {} -#endif +void InitTlsSize() {} +# endif void WriteToSyslog(const char *buffer) {} void Abort() { internal__exit(1); } bool CreateDir(const char *pathname) { return false; } -#endif // !SANITIZER_WINDOWS +#endif // !SANITIZER_WINDOWS #if !SANITIZER_WINDOWS && !SANITIZER_APPLE void ListOfModules::init() {} diff --git a/lib/tsan/sanitizer_common/sanitizer_common_syscalls.inc b/lib/tsan/sanitizer_common/sanitizer_common_syscalls.inc @@ -48,6 +48,7 @@ #if SANITIZER_LINUX # include "sanitizer_libc.h" +# include "sanitizer_platform_limits_posix.h" # define PRE_SYSCALL(name) \ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_pre_impl_##name @@ -2530,18 +2531,19 @@ PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) { # if !SANITIZER_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \ - defined(__loongarch__) || SANITIZER_RISCV64) - if (data) { + defined(__loongarch__) || SANITIZER_RISCV64 || defined(__sparc__)) + long data_arg = ptrace_data_arg(request, addr, data); + if (data_arg) { if (request == ptrace_setregs) { - PRE_READ((void *)data, struct_user_regs_struct_sz); + PRE_READ((void *)data_arg, struct_user_regs_struct_sz); } else if (request == ptrace_setfpregs) { - PRE_READ((void *)data, struct_user_fpregs_struct_sz); + PRE_READ((void *)data_arg, struct_user_fpregs_struct_sz); } else if (request == ptrace_setfpxregs) { - PRE_READ((void *)data, struct_user_fpxregs_struct_sz); + PRE_READ((void *)data_arg, struct_user_fpxregs_struct_sz); } else if (request == ptrace_setsiginfo) { - PRE_READ((void *)data, siginfo_t_sz); + PRE_READ((void *)data_arg, siginfo_t_sz); } else if (request == ptrace_setregset) { - __sanitizer_iovec *iov = (__sanitizer_iovec *)data; + __sanitizer_iovec *iov = (__sanitizer_iovec *)data_arg; PRE_READ(iov->iov_base, iov->iov_len); } } @@ -2552,25 +2554,26 @@ POST_SYSCALL(ptrace)(long res, long request, long pid, long addr, long data) { # if !SANITIZER_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \ - defined(__loongarch__) || SANITIZER_RISCV64) - if (res >= 0 && data) { + defined(__loongarch__) || SANITIZER_RISCV64 || defined(__sparc__)) + long data_arg = ptrace_data_arg(request, addr, data); + if (res >= 0 && data_arg) { // Note that this is different from the interceptor in // sanitizer_common_interceptors.inc. // PEEK* requests return resulting values through data pointer. if (request == ptrace_getregs) { - POST_WRITE((void *)data, struct_user_regs_struct_sz); + POST_WRITE((void *)data_arg, struct_user_regs_struct_sz); } else if (request == ptrace_getfpregs) { - POST_WRITE((void *)data, struct_user_fpregs_struct_sz); + POST_WRITE((void *)data_arg, struct_user_fpregs_struct_sz); } else if (request == ptrace_getfpxregs) { - POST_WRITE((void *)data, struct_user_fpxregs_struct_sz); + POST_WRITE((void *)data_arg, struct_user_fpxregs_struct_sz); } else if (request == ptrace_getsiginfo) { - POST_WRITE((void *)data, siginfo_t_sz); + POST_WRITE((void *)data_arg, siginfo_t_sz); } else if (request == ptrace_getregset) { - __sanitizer_iovec *iov = (__sanitizer_iovec *)data; + __sanitizer_iovec *iov = (__sanitizer_iovec *)data_arg; POST_WRITE(iov->iov_base, iov->iov_len); } else if (request == ptrace_peekdata || request == ptrace_peektext || request == ptrace_peekuser) { - POST_WRITE((void *)data, sizeof(void *)); + POST_WRITE((void *)data_arg, sizeof(void *)); } } # endif diff --git a/lib/tsan/sanitizer_common/sanitizer_coverage_win_dll_thunk.cpp b/lib/tsan/sanitizer_common/sanitizer_coverage_win_dll_thunk.cpp @@ -1,20 +0,0 @@ -//===-- sanitizer_coverage_win_dll_thunk.cpp ------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file defines a family of thunks that should be statically linked into -// the DLLs that have instrumentation in order to delegate the calls to the -// shared runtime that lives in the main binary. -// See https://github.com/google/sanitizers/issues/209 for the details. -//===----------------------------------------------------------------------===// -#ifdef SANITIZER_DLL_THUNK -#include "sanitizer_win_dll_thunk.h" -// Sanitizer Coverage interface functions. -#define INTERFACE_FUNCTION(Name) INTERCEPT_SANITIZER_FUNCTION(Name) -#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name) -#include "sanitizer_coverage_interface.inc" -#endif // SANITIZER_DLL_THUNK diff --git a/lib/tsan/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cpp b/lib/tsan/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cpp @@ -1,26 +0,0 @@ -//===-- sanitizer_coverage_win_dynamic_runtime_thunk.cpp ------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file defines things that need to be present in the application modules -// to interact with Sanitizer Coverage, when it is included in a dll. -// -//===----------------------------------------------------------------------===// -#ifdef SANITIZER_DYNAMIC_RUNTIME_THUNK -#define SANITIZER_IMPORT_INTERFACE 1 -#include "sanitizer_win_defs.h" -// Define weak alias for all weak functions imported from sanitizer coverage. -#define INTERFACE_FUNCTION(Name) -#define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name) -#include "sanitizer_coverage_interface.inc" -#endif // SANITIZER_DYNAMIC_RUNTIME_THUNK - -namespace __sanitizer { -// Add one, otherwise unused, external symbol to this object file so that the -// Visual C++ linker includes it and reads the .drective section. -void ForceWholeArchiveIncludeForSanCov() {} -} diff --git a/lib/tsan/sanitizer_common/sanitizer_coverage_win_weak_interception.cpp b/lib/tsan/sanitizer_common/sanitizer_coverage_win_weak_interception.cpp @@ -1,23 +0,0 @@ -//===-- sanitizer_coverage_win_weak_interception.cpp ----------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// This module should be included in Sanitizer Coverage when it implemented as a -// shared library on Windows (dll), in order to delegate the calls of weak -// functions to the implementation in the main executable when a strong -// definition is provided. -//===----------------------------------------------------------------------===// -#ifdef SANITIZER_DYNAMIC -#include "sanitizer_win_weak_interception.h" -#include "sanitizer_interface_internal.h" -#include "sancov_flags.h" -// Check if strong definitions for weak functions are present in the main -// executable. If that is the case, override dll functions to point to strong -// implementations. -#define INTERFACE_FUNCTION(Name) -#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name) -#include "sanitizer_coverage_interface.inc" -#endif // SANITIZER_DYNAMIC diff --git a/lib/tsan/sanitizer_common/sanitizer_deadlock_detector.h b/lib/tsan/sanitizer_common/sanitizer_deadlock_detector.h @@ -120,7 +120,7 @@ class DeadlockDetectorTLS { u32 lock; u32 stk; }; - LockWithContext all_locks_with_contexts_[64]; + LockWithContext all_locks_with_contexts_[128]; uptr n_all_locks_; }; diff --git a/lib/tsan/sanitizer_common/sanitizer_dense_map.h b/lib/tsan/sanitizer_common/sanitizer_dense_map.h @@ -69,24 +69,14 @@ class DenseMapBase { setNumTombstones(0); } + /// Return true if the specified key is in the map, false otherwise. + bool contains(const KeyT &Key) const { return doFind(Key) != nullptr; } + /// Return 1 if the specified key is in the map, 0 otherwise. - size_type count(const KeyT &Key) const { - const BucketT *TheBucket; - return LookupBucketFor(Key, TheBucket) ? 1 : 0; - } + size_type count(const KeyT &Key) const { return contains(Key) ? 1 : 0; } - value_type *find(const KeyT &Key) { - BucketT *TheBucket; - if (LookupBucketFor(Key, TheBucket)) - return TheBucket; - return nullptr; - } - const value_type *find(const KeyT &Key) const { - const BucketT *TheBucket; - if (LookupBucketFor(Key, TheBucket)) - return TheBucket; - return nullptr; - } + value_type *find(const KeyT &Key) { return doFind(Key); } + const value_type *find(const KeyT &Key) const { return doFind(Key); } /// Alternate version of find() which allows a different, and possibly /// less expensive, key type. @@ -95,25 +85,18 @@ class DenseMapBase { /// type used. template <class LookupKeyT> value_type *find_as(const LookupKeyT &Key) { - BucketT *TheBucket; - if (LookupBucketFor(Key, TheBucket)) - return TheBucket; - return nullptr; + return doFind(Key); } template <class LookupKeyT> const value_type *find_as(const LookupKeyT &Key) const { - const BucketT *TheBucket; - if (LookupBucketFor(Key, TheBucket)) - return TheBucket; - return nullptr; + return doFind(Key); } /// lookup - Return the entry for the specified key, or a default /// constructed value if no such entry exists. ValueT lookup(const KeyT &Key) const { - const BucketT *TheBucket; - if (LookupBucketFor(Key, TheBucket)) - return TheBucket->getSecond(); + if (const BucketT *Bucket = doFind(Key)) + return Bucket->getSecond(); return ValueT(); } @@ -184,8 +167,8 @@ class DenseMapBase { } bool erase(const KeyT &Val) { - BucketT *TheBucket; - if (!LookupBucketFor(Val, TheBucket)) + BucketT *TheBucket = doFind(Val); + if (!TheBucket) return false; // not in map. TheBucket->getSecond().~ValueT(); @@ -449,6 +432,35 @@ class DenseMapBase { return TheBucket; } + template <typename LookupKeyT> + BucketT *doFind(const LookupKeyT &Val) { + BucketT *BucketsPtr = getBuckets(); + const unsigned NumBuckets = getNumBuckets(); + if (NumBuckets == 0) + return nullptr; + + const KeyT EmptyKey = getEmptyKey(); + unsigned BucketNo = getHashValue(Val) & (NumBuckets - 1); + unsigned ProbeAmt = 1; + while (true) { + BucketT *Bucket = BucketsPtr + BucketNo; + if (LIKELY(KeyInfoT::isEqual(Val, Bucket->getFirst()))) + return Bucket; + if (LIKELY(KeyInfoT::isEqual(Bucket->getFirst(), EmptyKey))) + return nullptr; + + // Otherwise, it's a hash collision or a tombstone, continue quadratic + // probing. + BucketNo += ProbeAmt++; + BucketNo &= NumBuckets - 1; + } + } + + template <typename LookupKeyT> + const BucketT *doFind(const LookupKeyT &Val) const { + return const_cast<DenseMapBase *>(this)->doFind(Val); + } + /// LookupBucketFor - Lookup the appropriate bucket for Val, returning it in /// FoundBucket. If the bucket contains the key and a value, this returns /// true, otherwise it returns a bucket with an empty marker or tombstone and diff --git a/lib/tsan/sanitizer_common/sanitizer_errno.cpp b/lib/tsan/sanitizer_common/sanitizer_errno.cpp @@ -23,6 +23,7 @@ namespace __sanitizer { COMPILER_CHECK(errno_ENOMEM == ENOMEM); COMPILER_CHECK(errno_EBUSY == EBUSY); COMPILER_CHECK(errno_EINVAL == EINVAL); +COMPILER_CHECK(errno_ERANGE == ERANGE); // EOWNERDEAD is not present in some older platforms. #if defined(EOWNERDEAD) diff --git a/lib/tsan/sanitizer_common/sanitizer_errno_codes.h b/lib/tsan/sanitizer_common/sanitizer_errno_codes.h @@ -24,6 +24,7 @@ namespace __sanitizer { #define errno_ENOMEM 12 #define errno_EBUSY 16 #define errno_EINVAL 22 +#define errno_ERANGE 34 #define errno_ENAMETOOLONG 36 #define errno_ENOSYS 38 diff --git a/lib/tsan/sanitizer_common/sanitizer_fuchsia.cpp b/lib/tsan/sanitizer_common/sanitizer_fuchsia.cpp @@ -94,7 +94,6 @@ void DisableCoreDumperIfNecessary() {} void InstallDeadlySignalHandlers(SignalHandlerType handler) {} void SetAlternateSignalStack() {} void UnsetAlternateSignalStack() {} -void InitTlsSize() {} bool SignalContext::IsStackOverflow() const { return false; } void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); } @@ -445,6 +444,11 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) { return status == ZX_OK; } +bool TryMemCpy(void *dest, const void *src, uptr n) { + // TODO: implement. + return false; +} + // FIXME implement on this platform. void GetMemoryProfile(fill_profile_f cb, uptr *stats) {} @@ -519,7 +523,6 @@ uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) { uptr MainThreadStackBase, MainThreadStackSize; bool GetRandom(void *buffer, uptr length, bool blocking) { - CHECK_LE(length, ZX_CPRNG_DRAW_MAX_LEN); _zx_cprng_draw(buffer, length); return true; } diff --git a/lib/tsan/sanitizer_common/sanitizer_getauxval.h b/lib/tsan/sanitizer_common/sanitizer_getauxval.h @@ -21,22 +21,21 @@ #if SANITIZER_LINUX || SANITIZER_FUCHSIA -# if (__GLIBC_PREREQ(2, 16) || (SANITIZER_ANDROID && __ANDROID_API__ >= 21) || \ - SANITIZER_FUCHSIA) && \ - !SANITIZER_GO -# define SANITIZER_USE_GETAUXVAL 1 -# else -# define SANITIZER_USE_GETAUXVAL 0 -# endif - -# if SANITIZER_USE_GETAUXVAL -# include <sys/auxv.h> -# else +# if (__GLIBC_PREREQ(2, 16) || SANITIZER_ANDROID || SANITIZER_FUCHSIA) && \ + !SANITIZER_GO +# define SANITIZER_USE_GETAUXVAL 1 +# else +# define SANITIZER_USE_GETAUXVAL 0 +# endif + +# if SANITIZER_USE_GETAUXVAL +# include <sys/auxv.h> +# else // The weak getauxval definition allows to check for the function at runtime. // This is useful for Android, when compiled at a lower API level yet running // on a more recent platform that offers the function. extern "C" SANITIZER_WEAK_ATTRIBUTE unsigned long getauxval(unsigned long type); -# endif +# endif #elif SANITIZER_NETBSD diff --git a/lib/tsan/sanitizer_common/sanitizer_interface_internal.h b/lib/tsan/sanitizer_common/sanitizer_interface_internal.h @@ -49,6 +49,11 @@ __sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_report_error_summary(const char *error_summary); +// Returns size of dynamically allocated block. This function can be overridden +// by the client. +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE __sanitizer::uptr +__sanitizer_get_dtls_size(const void *tls_begin); + SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump(); SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage( const __sanitizer::uptr *pcs, const __sanitizer::uptr len); @@ -71,6 +76,11 @@ void __sanitizer_annotate_double_ended_contiguous_container( const void *old_container_beg, const void *old_container_end, const void *new_container_beg, const void *new_container_end); SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_copy_contiguous_container_annotations(const void *src_begin, + const void *src_end, + const void *dst_begin, + const void *dst_end); +SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_verify_contiguous_container(const void *beg, const void *mid, const void *end); SANITIZER_INTERFACE_ATTRIBUTE diff --git a/lib/tsan/sanitizer_common/sanitizer_internal_defs.h b/lib/tsan/sanitizer_common/sanitizer_internal_defs.h @@ -138,19 +138,25 @@ // in a portable way by the language itself. namespace __sanitizer { -#if defined(_WIN64) +#if defined(__UINTPTR_TYPE__) +# if defined(__arm__) && defined(__linux__) +// Linux Arm headers redefine __UINTPTR_TYPE__ and disagree with clang/gcc. +typedef unsigned int uptr; +typedef int sptr; +# else +typedef __UINTPTR_TYPE__ uptr; +typedef __INTPTR_TYPE__ sptr; +# endif +#elif defined(_WIN64) // 64-bit Windows uses LLP64 data model. typedef unsigned long long uptr; typedef signed long long sptr; -#else -# if (SANITIZER_WORDSIZE == 64) || SANITIZER_APPLE || SANITIZER_WINDOWS -typedef unsigned long uptr; -typedef signed long sptr; -# else +#elif defined(_WIN32) typedef unsigned int uptr; typedef signed int sptr; -# endif -#endif // defined(_WIN64) +#else +# error Unsupported compiler, missing __UINTPTR_TYPE__ +#endif // defined(__UINTPTR_TYPE__) #if defined(__x86_64__) // Since x32 uses ILP32 data model in 64-bit hardware mode, we must use // 64-bit pointer to unwind stack frame. @@ -194,16 +200,13 @@ typedef u64 OFF64_T; #ifdef __SIZE_TYPE__ typedef __SIZE_TYPE__ usize; #else -// Since we use this for operator new, usize must match the real size_t, but on -// 32-bit Windows the definition of uptr does not actually match uintptr_t or -// size_t because we are working around typedef mismatches for the (S)SIZE_T -// types used in interception.h. -// Until the definition of uptr has been fixed we have to special case Win32. -# if SANITIZER_WINDOWS && SANITIZER_WORDSIZE == 32 -typedef unsigned int usize; -# else typedef uptr usize; -# endif +#endif + +#if defined(__s390__) && !defined(__s390x__) +typedef long ssize; +#else +typedef sptr ssize; #endif typedef u64 tid_t; @@ -466,6 +469,9 @@ using namespace __sanitizer; namespace __msan { using namespace __sanitizer; } +namespace __nsan { +using namespace __sanitizer; +} namespace __hwasan { using namespace __sanitizer; } diff --git a/lib/tsan/sanitizer_common/sanitizer_libignore.cpp b/lib/tsan/sanitizer_common/sanitizer_libignore.cpp @@ -32,7 +32,7 @@ void LibIgnore::AddIgnoredLibrary(const char *name_templ) { lib->templ = internal_strdup(name_templ); lib->name = nullptr; lib->real_name = nullptr; - lib->loaded = false; + lib->range_id = kInvalidCodeRangeId; } void LibIgnore::OnLibraryLoaded(const char *name) { @@ -43,7 +43,7 @@ void LibIgnore::OnLibraryLoaded(const char *name) { buf[0]) { for (uptr i = 0; i < count_; i++) { Lib *lib = &libs_[i]; - if (!lib->loaded && (!lib->real_name) && + if (!lib->loaded() && (!lib->real_name) && TemplateMatch(lib->templ, name)) lib->real_name = internal_strdup(buf.data()); } @@ -70,28 +70,31 @@ void LibIgnore::OnLibraryLoaded(const char *name) { Die(); } loaded = true; - if (lib->loaded) + if (lib->loaded()) continue; VReport(1, "Matched called_from_lib suppression '%s' against library" " '%s'\n", lib->templ, mod.full_name()); - lib->loaded = true; lib->name = internal_strdup(mod.full_name()); const uptr idx = atomic_load(&ignored_ranges_count_, memory_order_relaxed); CHECK_LT(idx, ARRAY_SIZE(ignored_code_ranges_)); - ignored_code_ranges_[idx].begin = range.beg; - ignored_code_ranges_[idx].end = range.end; + ignored_code_ranges_[idx].OnLoad(range.beg, range.end); + // Record the index of the ignored range. + lib->range_id = idx; atomic_store(&ignored_ranges_count_, idx + 1, memory_order_release); break; } } - if (lib->loaded && !loaded) { - Report("%s: library '%s' that was matched against called_from_lib" - " suppression '%s' is unloaded\n", - SanitizerToolName, lib->name, lib->templ); - Die(); + if (lib->loaded() && !loaded) { + VReport(1, + "%s: library '%s' that was matched against called_from_lib" + " suppression '%s' is unloaded\n", + SanitizerToolName, lib->name, lib->templ); + // The library is unloaded so mark the ignored code range as unloaded. + ignored_code_ranges_[lib->range_id].OnUnload(); + lib->range_id = kInvalidCodeRangeId; } } @@ -110,8 +113,7 @@ void LibIgnore::OnLibraryLoaded(const char *name) { const uptr idx = atomic_load(&instrumented_ranges_count_, memory_order_relaxed); CHECK_LT(idx, ARRAY_SIZE(instrumented_code_ranges_)); - instrumented_code_ranges_[idx].begin = range.beg; - instrumented_code_ranges_[idx].end = range.end; + instrumented_code_ranges_[idx].OnLoad(range.beg, range.end); atomic_store(&instrumented_ranges_count_, idx + 1, memory_order_release); } diff --git a/lib/tsan/sanitizer_common/sanitizer_libignore.h b/lib/tsan/sanitizer_common/sanitizer_libignore.h @@ -49,25 +49,36 @@ class LibIgnore { bool IsPcInstrumented(uptr pc) const; private: + static const uptr kMaxIgnoredRanges = 128; + static const uptr kMaxInstrumentedRanges = 1024; + static const uptr kMaxLibs = 1024; + static const uptr kInvalidCodeRangeId = -1; + struct Lib { char *templ; char *name; char *real_name; // target of symlink - bool loaded; + uptr range_id; + bool loaded() const { return range_id != kInvalidCodeRangeId; }; }; struct LibCodeRange { - uptr begin; - uptr end; - }; + bool IsInRange(uptr pc) const { + return (pc >= begin && pc < atomic_load(&end, memory_order_acquire)); + } - inline bool IsInRange(uptr pc, const LibCodeRange &range) const { - return (pc >= range.begin && pc < range.end); - } + void OnLoad(uptr b, uptr e) { + begin = b; + atomic_store(&end, e, memory_order_release); + } - static const uptr kMaxIgnoredRanges = 128; - static const uptr kMaxInstrumentedRanges = 1024; - static const uptr kMaxLibs = 1024; + void OnUnload() { atomic_store(&end, 0, memory_order_release); } + + private: + uptr begin; + // A value of 0 means the associated module was unloaded. + atomic_uintptr_t end; + }; // Hot part: atomic_uintptr_t ignored_ranges_count_; @@ -90,7 +101,7 @@ class LibIgnore { inline bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const { const uptr n = atomic_load(&ignored_ranges_count_, memory_order_acquire); for (uptr i = 0; i < n; i++) { - if (IsInRange(pc, ignored_code_ranges_[i])) { + if (ignored_code_ranges_[i].IsInRange(pc)) { *pc_in_ignored_lib = true; return true; } @@ -104,7 +115,7 @@ inline bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const { inline bool LibIgnore::IsPcInstrumented(uptr pc) const { const uptr n = atomic_load(&instrumented_ranges_count_, memory_order_acquire); for (uptr i = 0; i < n; i++) { - if (IsInRange(pc, instrumented_code_ranges_[i])) + if (instrumented_code_ranges_[i].IsInRange(pc)) return true; } return false; diff --git a/lib/tsan/sanitizer_common/sanitizer_linux.cpp b/lib/tsan/sanitizer_common/sanitizer_linux.cpp @@ -107,7 +107,9 @@ extern struct ps_strings *__ps_strings; # endif // SANITIZER_NETBSD # if SANITIZER_SOLARIS +# include <stddef.h> # include <stdlib.h> +# include <sys/frame.h> # include <thread.h> # define environ _environ # endif @@ -132,9 +134,10 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE | FUTEX_PRIVATE_FLAG; // Are we using 32-bit or 64-bit Linux syscalls? // x32 (which defines __x86_64__) has SANITIZER_WORDSIZE == 32 // but it still needs to use 64-bit syscalls. -# if SANITIZER_LINUX && (defined(__x86_64__) || defined(__powerpc64__) || \ - SANITIZER_WORDSIZE == 64 || \ - (defined(__mips__) && _MIPS_SIM == _ABIN32)) +# if SANITIZER_LINUX && \ + (defined(__x86_64__) || defined(__powerpc64__) || \ + SANITIZER_WORDSIZE == 64 || \ + (defined(__mips__) && defined(_ABIN32) && _MIPS_SIM == _ABIN32)) # define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1 # else # define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0 @@ -152,6 +155,8 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE | FUTEX_PRIVATE_FLAG; # if SANITIZER_FREEBSD # define SANITIZER_USE_GETENTROPY 1 +extern "C" void *__sys_mmap(void *addr, size_t len, int prot, int flags, int fd, + off_t offset); # endif namespace __sanitizer { @@ -160,33 +165,56 @@ void SetSigProcMask(__sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, oldset)); } +# if SANITIZER_LINUX +// Deletes the specified signal from newset, if it is not present in oldset +// Equivalently: newset[signum] = newset[signum] & oldset[signum] +static void KeepUnblocked(__sanitizer_sigset_t &newset, + __sanitizer_sigset_t &oldset, int signum) { + // FIXME: https://github.com/google/sanitizers/issues/1816 + if (SANITIZER_ANDROID || !internal_sigismember(&oldset, signum)) + internal_sigdelset(&newset, signum); +} +# endif + // Block asynchronous signals void BlockSignals(__sanitizer_sigset_t *oldset) { - __sanitizer_sigset_t set; - internal_sigfillset(&set); -# if SANITIZER_LINUX && !SANITIZER_ANDROID + __sanitizer_sigset_t newset; + internal_sigfillset(&newset); + +# if SANITIZER_LINUX + __sanitizer_sigset_t currentset; + +# if !SANITIZER_ANDROID + // FIXME: https://github.com/google/sanitizers/issues/1816 + SetSigProcMask(NULL, ¤tset); + // Glibc uses SIGSETXID signal during setuid call. If this signal is blocked // on any thread, setuid call hangs. // See test/sanitizer_common/TestCases/Linux/setuid.c. - internal_sigdelset(&set, 33); -# endif -# if SANITIZER_LINUX + KeepUnblocked(newset, currentset, 33); +# endif // !SANITIZER_ANDROID + // Seccomp-BPF-sandboxed processes rely on SIGSYS to handle trapped syscalls. // If this signal is blocked, such calls cannot be handled and the process may // hang. - internal_sigdelset(&set, 31); + KeepUnblocked(newset, currentset, 31); +# if !SANITIZER_ANDROID // Don't block synchronous signals - internal_sigdelset(&set, SIGSEGV); - internal_sigdelset(&set, SIGBUS); - internal_sigdelset(&set, SIGILL); - internal_sigdelset(&set, SIGTRAP); - internal_sigdelset(&set, SIGABRT); - internal_sigdelset(&set, SIGFPE); - internal_sigdelset(&set, SIGPIPE); -# endif + // but also don't unblock signals that the user had deliberately blocked. + // FIXME: https://github.com/google/sanitizers/issues/1816 + KeepUnblocked(newset, currentset, SIGSEGV); + KeepUnblocked(newset, currentset, SIGBUS); + KeepUnblocked(newset, currentset, SIGILL); + KeepUnblocked(newset, currentset, SIGTRAP); + KeepUnblocked(newset, currentset, SIGABRT); + KeepUnblocked(newset, currentset, SIGFPE); + KeepUnblocked(newset, currentset, SIGPIPE); +# endif //! SANITIZER_ANDROID + +# endif // SANITIZER_LINUX - SetSigProcMask(&set, oldset); + SetSigProcMask(&newset, oldset); } ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) { @@ -218,7 +246,9 @@ ScopedBlockSignals::~ScopedBlockSignals() { SetSigProcMask(&saved_, nullptr); } # if !SANITIZER_S390 uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, u64 offset) { -# if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS +# if SANITIZER_FREEBSD + return (uptr)__sys_mmap(addr, length, prot, flags, fd, offset); +# elif SANITIZER_LINUX_USES_64BIT_SYSCALLS return internal_syscall(SYSCALL(mmap), (uptr)addr, length, prot, flags, fd, offset); # else @@ -250,6 +280,11 @@ int internal_madvise(uptr addr, uptr length, int advice) { return internal_syscall(SYSCALL(madvise), addr, length, advice); } +# if SANITIZER_FREEBSD +uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags) { + return internal_syscall(SYSCALL(close_range), lowfd, highfd, flags); +} +# endif uptr internal_close(fd_t fd) { return internal_syscall(SYSCALL(close), fd); } uptr internal_open(const char *filename, int flags) { @@ -395,8 +430,9 @@ uptr internal_stat(const char *path, void *buf) { AT_NO_AUTOMOUNT, STATX_BASIC_STATS, (uptr)&bufx); statx_to_stat(&bufx, (struct stat *)buf); return res; -# elif (SANITIZER_WORDSIZE == 64 || SANITIZER_X32 || \ - (defined(__mips__) && _MIPS_SIM == _ABIN32)) && \ +# elif ( \ + SANITIZER_WORDSIZE == 64 || SANITIZER_X32 || \ + (defined(__mips__) && defined(_ABIN32) && _MIPS_SIM == _ABIN32)) && \ !SANITIZER_SPARC return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0); @@ -433,8 +469,9 @@ uptr internal_lstat(const char *path, void *buf) { STATX_BASIC_STATS, (uptr)&bufx); statx_to_stat(&bufx, (struct stat *)buf); return res; -# elif (defined(_LP64) || SANITIZER_X32 || \ - (defined(__mips__) && _MIPS_SIM == _ABIN32)) && \ +# elif ( \ + defined(_LP64) || SANITIZER_X32 || \ + (defined(__mips__) && defined(_ABIN32) && _MIPS_SIM == _ABIN32)) && \ !SANITIZER_SPARC return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf, AT_SYMLINK_NOFOLLOW); @@ -721,6 +758,11 @@ static void GetArgsAndEnv(char ***argv, char ***envp) { # if !SANITIZER_GO if (&__libc_stack_end) { uptr *stack_end = (uptr *)__libc_stack_end; + // Linux/sparc64 needs an adjustment, cf. glibc + // sysdeps/sparc/sparc{32,64}/dl-machine.h (DL_STACK_END). +# if SANITIZER_LINUX && defined(__sparc__) + stack_end = &stack_end[16]; +# endif // Normally argc can be obtained from *stack_end, however, on ARM glibc's // _start clobbers it: // https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/arm/start.S;hb=refs/heads/release/2.31/master#l75 @@ -1014,34 +1056,29 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) { # if !SANITIZER_NETBSD // ThreadLister implementation. -ThreadLister::ThreadLister(pid_t pid) : pid_(pid), buffer_(4096) { - char task_directory_path[80]; - internal_snprintf(task_directory_path, sizeof(task_directory_path), - "/proc/%d/task/", pid); - descriptor_ = internal_open(task_directory_path, O_RDONLY | O_DIRECTORY); - if (internal_iserror(descriptor_)) { - Report("Can't open /proc/%d/task for reading.\n", pid); - } +ThreadLister::ThreadLister(pid_t pid) : buffer_(4096) { + task_path_.AppendF("/proc/%d/task", pid); } ThreadLister::Result ThreadLister::ListThreads( InternalMmapVector<tid_t> *threads) { - if (internal_iserror(descriptor_)) + int descriptor = internal_open(task_path_.data(), O_RDONLY | O_DIRECTORY); + if (internal_iserror(descriptor)) { + Report("Can't open %s for reading.\n", task_path_.data()); return Error; - internal_lseek(descriptor_, 0, SEEK_SET); + } + auto cleanup = at_scope_exit([&] { internal_close(descriptor); }); threads->clear(); Result result = Ok; for (bool first_read = true;; first_read = false) { - // Resize to max capacity if it was downsized by IsAlive. - buffer_.resize(buffer_.capacity()); CHECK_GE(buffer_.size(), 4096); uptr read = internal_getdents( - descriptor_, (struct linux_dirent *)buffer_.data(), buffer_.size()); + descriptor, (struct linux_dirent *)buffer_.data(), buffer_.size()); if (!read) return result; if (internal_iserror(read)) { - Report("Can't read directory entries from /proc/%d/task.\n", pid_); + Report("Can't read directory entries from %s.\n", task_path_.data()); return Error; } @@ -1079,26 +1116,33 @@ ThreadLister::Result ThreadLister::ListThreads( } } -bool ThreadLister::IsAlive(int tid) { +const char *ThreadLister::LoadStatus(tid_t tid) { + status_path_.clear(); + status_path_.AppendF("%s/%llu/status", task_path_.data(), tid); + auto cleanup = at_scope_exit([&] { + // Resize back to capacity if it is downsized by `ReadFileToVector`. + buffer_.resize(buffer_.capacity()); + }); + if (!ReadFileToVector(status_path_.data(), &buffer_) || buffer_.empty()) + return nullptr; + buffer_.push_back('\0'); + return buffer_.data(); +} + +bool ThreadLister::IsAlive(tid_t tid) { // /proc/%d/task/%d/status uses same call to detect alive threads as // proc_task_readdir. See task_state implementation in Linux. - char path[80]; - internal_snprintf(path, sizeof(path), "/proc/%d/task/%d/status", pid_, tid); - if (!ReadFileToVector(path, &buffer_) || buffer_.empty()) - return false; - buffer_.push_back(0); static const char kPrefix[] = "\nPPid:"; - const char *field = internal_strstr(buffer_.data(), kPrefix); + const char *status = LoadStatus(tid); + if (!status) + return false; + const char *field = internal_strstr(status, kPrefix); if (!field) return false; field += internal_strlen(kPrefix); return (int)internal_atoll(field) != 0; } -ThreadLister::~ThreadLister() { - if (!internal_iserror(descriptor_)) - internal_close(descriptor_); -} # endif # if SANITIZER_WORDSIZE == 32 @@ -1808,11 +1852,6 @@ int internal_uname(struct utsname *buf) { # endif # if SANITIZER_ANDROID -# if __ANDROID_API__ < 21 -extern "C" __attribute__((weak)) int dl_iterate_phdr( - int (*)(struct dl_phdr_info *, size_t, void *), void *); -# endif - static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size, void *data) { // Any name starting with "lib" indicates a bug in L where library base names @@ -1828,9 +1867,7 @@ static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size, static atomic_uint32_t android_api_level; static AndroidApiLevel AndroidDetectApiLevelStatic() { -# if __ANDROID_API__ <= 19 - return ANDROID_KITKAT; -# elif __ANDROID_API__ <= 22 +# if __ANDROID_API__ <= 22 return ANDROID_LOLLIPOP_MR1; # else return ANDROID_POST_LOLLIPOP; @@ -1838,8 +1875,6 @@ static AndroidApiLevel AndroidDetectApiLevelStatic() { } static AndroidApiLevel AndroidDetectApiLevel() { - if (!&dl_iterate_phdr) - return ANDROID_KITKAT; // K or lower bool base_name_seen = false; dl_iterate_phdr(dl_iterate_phdr_test_cb, &base_name_seen); if (base_name_seen) @@ -2014,6 +2049,18 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { return Unknown; return esr & ESR_ELx_WNR ? Write : Read; # elif defined(__loongarch__) + // In the musl environment, the Linux kernel uapi sigcontext.h is not + // included in signal.h. To avoid missing the SC_ADDRERR_{RD,WR} macros, + // copy them here. The LoongArch Linux kernel uapi is already stable, + // so there's no need to worry about the value changing. +# ifndef SC_ADDRERR_RD + // Address error was due to memory load +# define SC_ADDRERR_RD (1 << 30) +# endif +# ifndef SC_ADDRERR_WR + // Address error was due to memory store +# define SC_ADDRERR_WR (1 << 31) +# endif u32 flags = ucontext->uc_mcontext.__flags; if (flags & SC_ADDRERR_RD) return SignalContext::Read; @@ -2154,8 +2201,26 @@ bool SignalContext::IsTrueFaultingAddress() const { UNUSED static const char *RegNumToRegName(int reg) { switch (reg) { -# if SANITIZER_LINUX +# if SANITIZER_LINUX && SANITIZER_GLIBC || SANITIZER_NETBSD # if defined(__x86_64__) +# if SANITIZER_NETBSD +# define REG_RAX _REG_RAX +# define REG_RBX _REG_RBX +# define REG_RCX _REG_RCX +# define REG_RDX _REG_RDX +# define REG_RDI _REG_RDI +# define REG_RSI _REG_RSI +# define REG_RBP _REG_RBP +# define REG_RSP _REG_RSP +# define REG_R8 _REG_R8 +# define REG_R9 _REG_R9 +# define REG_R10 _REG_R10 +# define REG_R11 _REG_R11 +# define REG_R12 _REG_R12 +# define REG_R13 _REG_R13 +# define REG_R14 _REG_R14 +# define REG_R15 _REG_R15 +# endif case REG_RAX: return "rax"; case REG_RBX: @@ -2189,6 +2254,16 @@ static const char *RegNumToRegName(int reg) { case REG_R15: return "r15"; # elif defined(__i386__) +# if SANITIZER_NETBSD +# define REG_EAX _REG_EAX +# define REG_EBX _REG_EBX +# define REG_ECX _REG_ECX +# define REG_EDX _REG_EDX +# define REG_EDI _REG_EDI +# define REG_ESI _REG_ESI +# define REG_EBP _REG_EBP +# define REG_ESP _REG_ESP +# endif case REG_EAX: return "eax"; case REG_EBX: @@ -2205,32 +2280,170 @@ static const char *RegNumToRegName(int reg) { return "ebp"; case REG_ESP: return "esp"; +# elif defined(__arm__) +# ifdef MAKE_CASE +# undef MAKE_CASE +# endif +# define REG_STR(reg) #reg +# define MAKE_CASE(N) \ + case REG_R##N: \ + return REG_STR(r##N) + MAKE_CASE(0); + MAKE_CASE(1); + MAKE_CASE(2); + MAKE_CASE(3); + MAKE_CASE(4); + MAKE_CASE(5); + MAKE_CASE(6); + MAKE_CASE(7); + MAKE_CASE(8); + MAKE_CASE(9); + MAKE_CASE(10); + MAKE_CASE(11); + MAKE_CASE(12); + case REG_R13: + return "sp"; + case REG_R14: + return "lr"; + case REG_R15: + return "pc"; +# elif defined(__aarch64__) +# define REG_STR(reg) #reg +# define MAKE_CASE(N) \ + case N: \ + return REG_STR(x##N) + MAKE_CASE(0); + MAKE_CASE(1); + MAKE_CASE(2); + MAKE_CASE(3); + MAKE_CASE(4); + MAKE_CASE(5); + MAKE_CASE(6); + MAKE_CASE(7); + MAKE_CASE(8); + MAKE_CASE(9); + MAKE_CASE(10); + MAKE_CASE(11); + MAKE_CASE(12); + MAKE_CASE(13); + MAKE_CASE(14); + MAKE_CASE(15); + MAKE_CASE(16); + MAKE_CASE(17); + MAKE_CASE(18); + MAKE_CASE(19); + MAKE_CASE(20); + MAKE_CASE(21); + MAKE_CASE(22); + MAKE_CASE(23); + MAKE_CASE(24); + MAKE_CASE(25); + MAKE_CASE(26); + MAKE_CASE(27); + MAKE_CASE(28); + case 29: + return "fp"; + case 30: + return "lr"; + case 31: + return "sp"; # endif -# endif +# endif // SANITIZER_LINUX && SANITIZER_GLIBC default: return NULL; } return NULL; } -# if SANITIZER_LINUX +# if ((SANITIZER_LINUX && SANITIZER_GLIBC) || SANITIZER_NETBSD) && \ + (defined(__arm__) || defined(__aarch64__)) +static uptr GetArmRegister(ucontext_t *ctx, int RegNum) { + switch (RegNum) { +# if defined(__arm__) && !SANITIZER_NETBSD +# ifdef MAKE_CASE +# undef MAKE_CASE +# endif +# define MAKE_CASE(N) \ + case REG_R##N: \ + return ctx->uc_mcontext.arm_r##N + MAKE_CASE(0); + MAKE_CASE(1); + MAKE_CASE(2); + MAKE_CASE(3); + MAKE_CASE(4); + MAKE_CASE(5); + MAKE_CASE(6); + MAKE_CASE(7); + MAKE_CASE(8); + MAKE_CASE(9); + MAKE_CASE(10); + case REG_R11: + return ctx->uc_mcontext.arm_fp; + case REG_R12: + return ctx->uc_mcontext.arm_ip; + case REG_R13: + return ctx->uc_mcontext.arm_sp; + case REG_R14: + return ctx->uc_mcontext.arm_lr; + case REG_R15: + return ctx->uc_mcontext.arm_pc; +# elif defined(__aarch64__) +# if SANITIZER_LINUX + case 0 ... 30: + return ctx->uc_mcontext.regs[RegNum]; + case 31: + return ctx->uc_mcontext.sp; +# elif SANITIZER_NETBSD + case 0 ... 31: + return ctx->uc_mcontext.__gregs[RegNum]; +# endif +# endif + default: + return 0; + } + return 0; +} +# endif // SANITIZER_LINUX && SANITIZER_GLIBC && (defined(__arm__) || + // defined(__aarch64__)) + UNUSED static void DumpSingleReg(ucontext_t *ctx, int RegNum) { const char *RegName = RegNumToRegName(RegNum); +# if SANITIZER_LINUX && SANITIZER_GLIBC || SANITIZER_NETBSD # if defined(__x86_64__) Printf("%s%s = 0x%016llx ", internal_strlen(RegName) == 2 ? " " : "", - RegName, ctx->uc_mcontext.gregs[RegNum]); + RegName, +# if SANITIZER_LINUX + ctx->uc_mcontext.gregs[RegNum] +# elif SANITIZER_NETBSD + ctx->uc_mcontext.__gregs[RegNum] +# endif + ); # elif defined(__i386__) - Printf("%s = 0x%08x ", RegName, ctx->uc_mcontext.gregs[RegNum]); + Printf("%s = 0x%08x ", RegName, +# if SANITIZER_LINUX + ctx->uc_mcontext.gregs[RegNum] +# elif SANITIZER_NETBSD + ctx->uc_mcontext.__gregs[RegNum] +# endif + ); +# elif defined(__arm__) + Printf("%s%s = 0x%08zx ", internal_strlen(RegName) == 2 ? " " : "", RegName, + GetArmRegister(ctx, RegNum)); +# elif defined(__aarch64__) + Printf("%s%s = 0x%016zx ", internal_strlen(RegName) == 2 ? " " : "", RegName, + GetArmRegister(ctx, RegNum)); # else (void)RegName; # endif -} +# else + (void)RegName; # endif +} void SignalContext::DumpAllRegisters(void *context) { ucontext_t *ucontext = (ucontext_t *)context; -# if SANITIZER_LINUX +# if SANITIZER_LINUX && SANITIZER_GLIBC || SANITIZER_NETBSD # if defined(__x86_64__) Report("Register values:\n"); DumpSingleReg(ucontext, REG_RAX); @@ -2269,6 +2482,35 @@ void SignalContext::DumpAllRegisters(void *context) { DumpSingleReg(ucontext, REG_EBP); DumpSingleReg(ucontext, REG_ESP); Printf("\n"); +# elif defined(__arm__) && !SANITIZER_NETBSD + Report("Register values:\n"); + DumpSingleReg(ucontext, REG_R0); + DumpSingleReg(ucontext, REG_R1); + DumpSingleReg(ucontext, REG_R2); + DumpSingleReg(ucontext, REG_R3); + Printf("\n"); + DumpSingleReg(ucontext, REG_R4); + DumpSingleReg(ucontext, REG_R5); + DumpSingleReg(ucontext, REG_R6); + DumpSingleReg(ucontext, REG_R7); + Printf("\n"); + DumpSingleReg(ucontext, REG_R8); + DumpSingleReg(ucontext, REG_R9); + DumpSingleReg(ucontext, REG_R10); + DumpSingleReg(ucontext, REG_R11); + Printf("\n"); + DumpSingleReg(ucontext, REG_R12); + DumpSingleReg(ucontext, REG_R13); + DumpSingleReg(ucontext, REG_R14); + DumpSingleReg(ucontext, REG_R15); + Printf("\n"); +# elif defined(__aarch64__) + Report("Register values:\n"); + for (int i = 0; i <= 31; ++i) { + DumpSingleReg(ucontext, i); + if (i % 4 == 3) + Printf("\n"); + } # else (void)ucontext; # endif @@ -2310,6 +2552,8 @@ void SignalContext::DumpAllRegisters(void *context) { # else (void)ucontext; # endif +# else + (void)ucontext; # endif // FIXME: Implement this for other OSes and architectures. } @@ -2404,7 +2648,19 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { # if SANITIZER_SOLARIS ucontext_t *ucontext = (ucontext_t *)context; *pc = ucontext->uc_mcontext.gregs[REG_PC]; - *sp = ucontext->uc_mcontext.gregs[REG_O6] + STACK_BIAS; + *sp = ucontext->uc_mcontext.gregs[REG_SP] + STACK_BIAS; + // Avoid SEGV when dereferencing sp on stack overflow with non-faulting load. + // This requires a SPARC V9 CPU. Cannot use #ASI_PNF here: only supported + // since clang-19. +# if defined(__sparcv9) + asm("ldxa [%[fp]] 0x82, %[bp]" +# else + asm("lduwa [%[fp]] 0x82, %[bp]" +# endif + : [bp] "=r"(*bp) + : [fp] "r"(&((struct frame *)*sp)->fr_savfp)); + if (*bp) + *bp += STACK_BIAS; # else // Historical BSDism here. struct sigcontext *scontext = (struct sigcontext *)context; @@ -2415,8 +2671,8 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { *pc = scontext->si_regs.pc; *sp = scontext->si_regs.u_regs[14]; # endif -# endif *bp = (uptr)((uhwptr *)*sp)[14] + STACK_BIAS; +# endif # elif defined(__mips__) ucontext_t *ucontext = (ucontext_t *)context; *pc = ucontext->uc_mcontext.pc; @@ -2459,9 +2715,7 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); } -void InitializePlatformEarly() { - // Do nothing. -} +void InitializePlatformEarly() { InitTlsSize(); } void CheckASLR() { # if SANITIZER_NETBSD diff --git a/lib/tsan/sanitizer_common/sanitizer_linux.h b/lib/tsan/sanitizer_common/sanitizer_linux.h @@ -97,19 +97,19 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg); class ThreadLister { public: explicit ThreadLister(pid_t pid); - ~ThreadLister(); enum Result { Error, Incomplete, Ok, }; Result ListThreads(InternalMmapVector<tid_t> *threads); + const char *LoadStatus(tid_t tid); private: - bool IsAlive(int tid); + bool IsAlive(tid_t tid); - pid_t pid_; - int descriptor_ = -1; + InternalScopedString task_path_; + InternalScopedString status_path_; InternalMmapVector<char> buffer_; }; diff --git a/lib/tsan/sanitizer_common/sanitizer_linux_libcdep.cpp b/lib/tsan/sanitizer_common/sanitizer_linux_libcdep.cpp @@ -40,6 +40,10 @@ # include <sys/resource.h> # include <syslog.h> +# if SANITIZER_GLIBC +# include <gnu/libc-version.h> +# endif + # if !defined(ElfW) # define ElfW(type) Elf_##type # endif @@ -53,7 +57,7 @@ // that, it was never implemented. So just define it to zero. # undef MAP_NORESERVE # define MAP_NORESERVE 0 -extern const Elf_Auxinfo *__elf_aux_vector; +extern const Elf_Auxinfo *__elf_aux_vector __attribute__((weak)); extern "C" int __sys_sigaction(int signum, const struct sigaction *act, struct sigaction *oldact); # endif @@ -196,27 +200,6 @@ bool SetEnv(const char *name, const char *value) { } # endif -__attribute__((unused)) static bool GetLibcVersion(int *major, int *minor, - int *patch) { -# ifdef _CS_GNU_LIBC_VERSION - char buf[64]; - uptr len = confstr(_CS_GNU_LIBC_VERSION, buf, sizeof(buf)); - if (len >= sizeof(buf)) - return false; - buf[len] = 0; - static const char kGLibC[] = "glibc "; - if (internal_strncmp(buf, kGLibC, sizeof(kGLibC) - 1) != 0) - return false; - const char *p = buf + sizeof(kGLibC) - 1; - *major = internal_simple_strtoll(p, &p, 10); - *minor = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0; - *patch = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0; - return true; -# else - return false; -# endif -} - // True if we can use dlpi_tls_data. glibc before 2.25 may leave NULL (BZ // #19826) so dlpi_tls_data cannot be used. // @@ -226,112 +209,166 @@ __attribute__((unused)) static bool GetLibcVersion(int *major, int *minor, __attribute__((unused)) static int g_use_dlpi_tls_data; # if SANITIZER_GLIBC && !SANITIZER_GO -__attribute__((unused)) static size_t g_tls_size; -void InitTlsSize() { - int major, minor, patch; - g_use_dlpi_tls_data = - GetLibcVersion(&major, &minor, &patch) && major == 2 && minor >= 25; - -# if defined(__aarch64__) || defined(__x86_64__) || \ - defined(__powerpc64__) || defined(__loongarch__) - void *get_tls_static_info = dlsym(RTLD_NEXT, "_dl_get_tls_static_info"); - size_t tls_align; - ((void (*)(size_t *, size_t *))get_tls_static_info)(&g_tls_size, &tls_align); -# endif +static void GetGLibcVersion(int *major, int *minor, int *patch) { + const char *p = gnu_get_libc_version(); + *major = internal_simple_strtoll(p, &p, 10); + // Caller does not expect anything else. + CHECK_EQ(*major, 2); + *minor = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0; + *patch = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0; } -# else -void InitTlsSize() {} -# endif // SANITIZER_GLIBC && !SANITIZER_GO - -// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage -// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan -// to get the pointer to thread-specific data keys in the thread control block. -# if (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS) && \ - !SANITIZER_ANDROID && !SANITIZER_GO -// sizeof(struct pthread) from glibc. -static atomic_uintptr_t thread_descriptor_size; static uptr ThreadDescriptorSizeFallback() { - uptr val = 0; -# if defined(__x86_64__) || defined(__i386__) || defined(__arm__) +# if defined(__x86_64__) || defined(__i386__) || defined(__arm__) || \ + SANITIZER_RISCV64 int major; int minor; int patch; - if (GetLibcVersion(&major, &minor, &patch) && major == 2) { - /* sizeof(struct pthread) values from various glibc versions. */ - if (SANITIZER_X32) - val = 1728; // Assume only one particular version for x32. - // For ARM sizeof(struct pthread) changed in Glibc 2.23. - else if (SANITIZER_ARM) - val = minor <= 22 ? 1120 : 1216; - else if (minor <= 3) - val = FIRST_32_SECOND_64(1104, 1696); - else if (minor == 4) - val = FIRST_32_SECOND_64(1120, 1728); - else if (minor == 5) - val = FIRST_32_SECOND_64(1136, 1728); - else if (minor <= 9) - val = FIRST_32_SECOND_64(1136, 1712); - else if (minor == 10) - val = FIRST_32_SECOND_64(1168, 1776); - else if (minor == 11 || (minor == 12 && patch == 1)) - val = FIRST_32_SECOND_64(1168, 2288); - else if (minor <= 14) - val = FIRST_32_SECOND_64(1168, 2304); - else if (minor < 32) // Unknown version - val = FIRST_32_SECOND_64(1216, 2304); - else // minor == 32 - val = FIRST_32_SECOND_64(1344, 2496); - } -# elif defined(__s390__) || defined(__sparc__) + GetGLibcVersion(&major, &minor, &patch); +# endif + +# if defined(__x86_64__) || defined(__i386__) || defined(__arm__) + /* sizeof(struct pthread) values from various glibc versions. */ + if (SANITIZER_X32) + return 1728; // Assume only one particular version for x32. + // For ARM sizeof(struct pthread) changed in Glibc 2.23. + if (SANITIZER_ARM) + return minor <= 22 ? 1120 : 1216; + if (minor <= 3) + return FIRST_32_SECOND_64(1104, 1696); + if (minor == 4) + return FIRST_32_SECOND_64(1120, 1728); + if (minor == 5) + return FIRST_32_SECOND_64(1136, 1728); + if (minor <= 9) + return FIRST_32_SECOND_64(1136, 1712); + if (minor == 10) + return FIRST_32_SECOND_64(1168, 1776); + if (minor == 11 || (minor == 12 && patch == 1)) + return FIRST_32_SECOND_64(1168, 2288); + if (minor <= 14) + return FIRST_32_SECOND_64(1168, 2304); + if (minor < 32) // Unknown version + return FIRST_32_SECOND_64(1216, 2304); + // minor == 32 + return FIRST_32_SECOND_64(1344, 2496); +# endif + +# if SANITIZER_RISCV64 + // TODO: consider adding an optional runtime check for an unknown (untested) + // glibc version + if (minor <= 28) // WARNING: the highest tested version is 2.29 + return 1772; // no guarantees for this one + if (minor <= 31) + return 1772; // tested against glibc 2.29, 2.31 + return 1936; // tested against glibc 2.32 +# endif + +# if defined(__s390__) || defined(__sparc__) // The size of a prefix of TCB including pthread::{specific_1stblock,specific} // suffices. Just return offsetof(struct pthread, specific_used), which hasn't // changed since 2007-05. Technically this applies to i386/x86_64 as well but // we call _dl_get_tls_static_info and need the precise size of struct // pthread. return FIRST_32_SECOND_64(524, 1552); -# elif defined(__mips__) +# endif + +# if defined(__mips__) // TODO(sagarthakur): add more values as per different glibc versions. - val = FIRST_32_SECOND_64(1152, 1776); -# elif SANITIZER_LOONGARCH64 - val = 1856; // from glibc 2.36 -# elif SANITIZER_RISCV64 - int major; - int minor; - int patch; - if (GetLibcVersion(&major, &minor, &patch) && major == 2) { - // TODO: consider adding an optional runtime check for an unknown (untested) - // glibc version - if (minor <= 28) // WARNING: the highest tested version is 2.29 - val = 1772; // no guarantees for this one - else if (minor <= 31) - val = 1772; // tested against glibc 2.29, 2.31 - else - val = 1936; // tested against glibc 2.32 - } + return FIRST_32_SECOND_64(1152, 1776); +# endif + +# if SANITIZER_LOONGARCH64 + return 1856; // from glibc 2.36 +# endif -# elif defined(__aarch64__) +# if defined(__aarch64__) // The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22. - val = 1776; -# elif defined(__powerpc64__) - val = 1776; // from glibc.ppc64le 2.20-8.fc21 + return 1776; +# endif + +# if defined(__powerpc64__) + return 1776; // from glibc.ppc64le 2.20-8.fc21 # endif - return val; } +# endif // SANITIZER_GLIBC && !SANITIZER_GO + +# if SANITIZER_FREEBSD && !SANITIZER_GO +// FIXME: Implementation is very GLIBC specific, but it's used by FreeBSD. +static uptr ThreadDescriptorSizeFallback() { +# if defined(__s390__) || defined(__sparc__) + // The size of a prefix of TCB including pthread::{specific_1stblock,specific} + // suffices. Just return offsetof(struct pthread, specific_used), which hasn't + // changed since 2007-05. Technically this applies to i386/x86_64 as well but + // we call _dl_get_tls_static_info and need the precise size of struct + // pthread. + return FIRST_32_SECOND_64(524, 1552); +# endif -uptr ThreadDescriptorSize() { - uptr val = atomic_load_relaxed(&thread_descriptor_size); - if (val) - return val; - // _thread_db_sizeof_pthread is a GLIBC_PRIVATE symbol that is exported in - // glibc 2.34 and later. - if (unsigned *psizeof = static_cast<unsigned *>( - dlsym(RTLD_DEFAULT, "_thread_db_sizeof_pthread"))) - val = *psizeof; - if (!val) - val = ThreadDescriptorSizeFallback(); - atomic_store_relaxed(&thread_descriptor_size, val); - return val; +# if defined(__mips__) + // TODO(sagarthakur): add more values as per different glibc versions. + return FIRST_32_SECOND_64(1152, 1776); +# endif + +# if SANITIZER_LOONGARCH64 + return 1856; // from glibc 2.36 +# endif + +# if defined(__aarch64__) + // The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22. + return 1776; +# endif + +# if defined(__powerpc64__) + return 1776; // from glibc.ppc64le 2.20-8.fc21 +# endif + + return 0; +} +# endif // SANITIZER_FREEBSD && !SANITIZER_GO + +# if (SANITIZER_FREEBSD || SANITIZER_GLIBC) && !SANITIZER_GO +// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage +// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan +// to get the pointer to thread-specific data keys in the thread control block. +// sizeof(struct pthread) from glibc. +static uptr thread_descriptor_size; + +uptr ThreadDescriptorSize() { return thread_descriptor_size; } + +# if SANITIZER_GLIBC +__attribute__((unused)) static size_t g_tls_size; +# endif + +void InitTlsSize() { +# if SANITIZER_GLIBC + int major, minor, patch; + GetGLibcVersion(&major, &minor, &patch); + g_use_dlpi_tls_data = major == 2 && minor >= 25; + + if (major == 2 && minor >= 34) { + // _thread_db_sizeof_pthread is a GLIBC_PRIVATE symbol that is exported in + // glibc 2.34 and later. + if (unsigned *psizeof = static_cast<unsigned *>( + dlsym(RTLD_DEFAULT, "_thread_db_sizeof_pthread"))) { + thread_descriptor_size = *psizeof; + } + } + +# if defined(__aarch64__) || defined(__x86_64__) || \ + defined(__powerpc64__) || defined(__loongarch__) + auto *get_tls_static_info = (void (*)(size_t *, size_t *))dlsym( + RTLD_DEFAULT, "_dl_get_tls_static_info"); + size_t tls_align; + // Can be null if static link. + if (get_tls_static_info) + get_tls_static_info(&g_tls_size, &tls_align); +# endif + +# endif // SANITIZER_GLIBC + + if (!thread_descriptor_size) + thread_descriptor_size = ThreadDescriptorSizeFallback(); } # if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64 || \ @@ -354,7 +391,13 @@ static uptr TlsPreTcbSize() { return kTlsPreTcbSize; } # endif +# else // (SANITIZER_FREEBSD || SANITIZER_GLIBC) && !SANITIZER_GO +void InitTlsSize() {} +uptr ThreadDescriptorSize() { return 0; } +# endif // (SANITIZER_FREEBSD || SANITIZER_GLIBC) && !SANITIZER_GO +# if (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS) && \ + !SANITIZER_ANDROID && !SANITIZER_GO namespace { struct TlsBlock { uptr begin, end, align; @@ -626,25 +669,32 @@ uptr GetTlsSize() { } # endif -void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, - uptr *tls_addr, uptr *tls_size) { +void GetThreadStackAndTls(bool main, uptr *stk_begin, uptr *stk_end, + uptr *tls_begin, uptr *tls_end) { # if SANITIZER_GO // Stub implementation for Go. - *stk_addr = *stk_size = *tls_addr = *tls_size = 0; + *stk_begin = 0; + *stk_end = 0; + *tls_begin = 0; + *tls_end = 0; # else - GetTls(tls_addr, tls_size); + uptr tls_addr = 0; + uptr tls_size = 0; + GetTls(&tls_addr, &tls_size); + *tls_begin = tls_addr; + *tls_end = tls_addr + tls_size; uptr stack_top, stack_bottom; GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom); - *stk_addr = stack_bottom; - *stk_size = stack_top - stack_bottom; + *stk_begin = stack_bottom; + *stk_end = stack_top; if (!main) { // If stack and tls intersect, make them non-intersecting. - if (*tls_addr > *stk_addr && *tls_addr < *stk_addr + *stk_size) { - if (*stk_addr + *stk_size < *tls_addr + *tls_size) - *tls_size = *stk_addr + *stk_size - *tls_addr; - *stk_size = *tls_addr - *stk_addr; + if (*tls_begin > *stk_begin && *tls_begin < *stk_end) { + if (*stk_end < *tls_end) + *tls_end = *stk_end; + *stk_end = *tls_begin; } } # endif @@ -723,11 +773,6 @@ static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) { return 0; } -# if SANITIZER_ANDROID && __ANDROID_API__ < 21 -extern "C" __attribute__((weak)) int dl_iterate_phdr( - int (*)(struct dl_phdr_info *, size_t, void *), void *); -# endif - static bool requiresProcmaps() { # if SANITIZER_ANDROID && __ANDROID_API__ <= 22 // Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken. @@ -890,11 +935,8 @@ extern "C" SANITIZER_WEAK_ATTRIBUTE int __android_log_write(int prio, void WriteOneLineToSyslog(const char *s) { if (&async_safe_write_log) { async_safe_write_log(SANITIZER_ANDROID_LOG_INFO, GetProcessName(), s); - } else if (AndroidGetApiLevel() > ANDROID_KITKAT) { - syslog(LOG_INFO, "%s", s); } else { - CHECK(&__android_log_write); - __android_log_write(SANITIZER_ANDROID_LOG_INFO, nullptr, s); + syslog(LOG_INFO, "%s", s); } } diff --git a/lib/tsan/sanitizer_common/sanitizer_mac.cpp b/lib/tsan/sanitizer_common/sanitizer_mac.cpp @@ -45,7 +45,7 @@ extern char **environ; # define SANITIZER_OS_TRACE 0 # endif -// import new crash reporting api +// Integrate with CrashReporter library if available # if defined(__has_include) && __has_include(<CrashReporterClient.h>) # define HAVE_CRASHREPORTERCLIENT_H 1 # include <CrashReporterClient.h> @@ -545,9 +545,6 @@ uptr GetTlsSize() { return 0; } -void InitTlsSize() { -} - uptr TlsBaseAddr() { uptr segbase = 0; #if defined(__x86_64__) @@ -572,21 +569,18 @@ uptr TlsSize() { #endif } -void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, - uptr *tls_addr, uptr *tls_size) { -#if !SANITIZER_GO - uptr stack_top, stack_bottom; - GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom); - *stk_addr = stack_bottom; - *stk_size = stack_top - stack_bottom; - *tls_addr = TlsBaseAddr(); - *tls_size = TlsSize(); -#else - *stk_addr = 0; - *stk_size = 0; - *tls_addr = 0; - *tls_size = 0; -#endif +void GetThreadStackAndTls(bool main, uptr *stk_begin, uptr *stk_end, + uptr *tls_begin, uptr *tls_end) { +# if !SANITIZER_GO + GetThreadStackTopAndBottom(main, stk_end, stk_begin); + *tls_begin = TlsBaseAddr(); + *tls_end = *tls_begin + TlsSize(); +# else + *stk_begin = 0; + *stk_end = 0; + *tls_begin = 0; + *tls_end = 0; +# endif } void ListOfModules::init() { @@ -788,7 +782,11 @@ void WriteOneLineToSyslog(const char *s) { if (GetMacosAlignedVersion() >= MacosVersion(10, 12)) { os_log_error(OS_LOG_DEFAULT, "%{public}s", s); } else { +#pragma clang diagnostic push +// as_log is deprecated. +#pragma clang diagnostic ignored "-Wdeprecated-declarations" asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "%s", s); +#pragma clang diagnostic pop } #endif } @@ -798,8 +796,13 @@ static char crashreporter_info_buff[__sanitizer::kErrorMessageBufferSize] = {}; static Mutex crashreporter_info_mutex; extern "C" { -// Integrate with crash reporter libraries. + #if HAVE_CRASHREPORTERCLIENT_H +// Available in CRASHREPORTER_ANNOTATIONS_VERSION 5+ +# ifdef CRASHREPORTER_ANNOTATIONS_INITIALIZER +CRASHREPORTER_ANNOTATIONS_INITIALIZER() +# else +// Support for older CrashRerporter annotiations CRASH_REPORTER_CLIENT_HIDDEN struct crashreporter_annotations_t gCRAnnotations __attribute__((section("__DATA," CRASHREPORTER_ANNOTATIONS_SECTION))) = { @@ -810,17 +813,17 @@ struct crashreporter_annotations_t gCRAnnotations 0, 0, 0, -#if CRASHREPORTER_ANNOTATIONS_VERSION > 4 +# if CRASHREPORTER_ANNOTATIONS_VERSION > 4 0, -#endif +# endif }; - -#else -// fall back to old crashreporter api +# endif +# else +// Revert to previous crash reporter API if client header is not available static const char *__crashreporter_info__ __attribute__((__used__)) = &crashreporter_info_buff[0]; asm(".desc ___crashreporter_info__, 0x10"); -#endif +#endif // HAVE_CRASHREPORTERCLIENT_H } // extern "C" @@ -843,6 +846,9 @@ void LogFullErrorReport(const char *buffer) { #if !SANITIZER_GO // Log with os_trace. This will make it into the crash log. #if SANITIZER_OS_TRACE +#pragma clang diagnostic push +// os_trace is deprecated. +#pragma clang diagnostic ignored "-Wdeprecated-declarations" if (GetMacosAlignedVersion() >= MacosVersion(10, 10)) { // os_trace requires the message (format parameter) to be a string literal. if (internal_strncmp(SanitizerToolName, "AddressSanitizer", @@ -860,6 +866,7 @@ void LogFullErrorReport(const char *buffer) { if (common_flags()->log_to_syslog) os_trace("Consult syslog for more information."); } +#pragma clang diagnostic pop #endif // Log to syslog. @@ -970,8 +977,9 @@ static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES"; LowLevelAllocator allocator_for_env; static bool ShouldCheckInterceptors() { - // Restrict "interceptors working?" check to ASan and TSan. - const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer"}; + // Restrict "interceptors working?" check + const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer", + "RealtimeSanitizer"}; size_t count = sizeof(sanitizer_names) / sizeof(sanitizer_names[0]); for (size_t i = 0; i < count; i++) { if (internal_strcmp(sanitizer_names[i], SanitizerToolName) == 0) diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_interceptors.h b/lib/tsan/sanitizer_common/sanitizer_platform_interceptors.h @@ -84,6 +84,25 @@ #define SI_NOT_MAC 1 #endif +#if SANITIZER_APPLE +# include <Availability.h> + +// aligned_alloc was introduced in OSX 10.15 +// Linking will fail when using an older SDK +# if defined(__MAC_10_15) +// macOS 10.15 is greater than our minimal deployment target. To ensure we +// generate a weak reference so the dylib continues to work on older +// systems, we need to forward declare the intercepted function as "weak +// imports". +SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment, + __sanitizer::usize __size); +# define SI_MAC_SDK_10_15_AVAILABLE 1 +# else +# define SI_MAC_SDK_10_15_AVAILABLE 0 +# endif // defined(__MAC_10_15) + +#endif // SANITIZER_APPLE + #if SANITIZER_IOS #define SI_IOS 1 #else @@ -183,9 +202,16 @@ #define SANITIZER_INTERCEPT_FPUTS SI_POSIX #define SANITIZER_INTERCEPT_PUTS SI_POSIX +#define SANITIZER_INTERCEPT_CREAT64 (SI_GLIBC || SI_SOLARIS32) +#define SANITIZER_INTERCEPT_FCNTL64 (SI_GLIBC || SI_SOLARIS32) +#define SANITIZER_INTERCEPT_OPEN64 (SI_GLIBC || SI_SOLARIS32) +#define SANITIZER_INTERCEPT_OPENAT64 (SI_GLIBC || SI_SOLARIS32) + #define SANITIZER_INTERCEPT_PREAD64 (SI_GLIBC || SI_SOLARIS32) #define SANITIZER_INTERCEPT_PWRITE64 (SI_GLIBC || SI_SOLARIS32) +#define SANITIZER_INTERCEPT_LSEEK64 (SI_GLIBC || SI_SOLARIS32) + #define SANITIZER_INTERCEPT_READV SI_POSIX #define SANITIZER_INTERCEPT_WRITEV SI_POSIX @@ -232,8 +258,12 @@ (SI_FREEBSD || SI_NETBSD || SI_LINUX || SI_SOLARIS) #define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID \ (SI_LINUX || SI_FREEBSD || SI_NETBSD) +// TODO: This should be SI_POSIX, adding glibc first until I have time +// to verify all timer_t typedefs on other platforms. +#define SANITIZER_INTERCEPT_TIMER_CREATE SI_GLIBC #define SANITIZER_INTERCEPT_GETITIMER SI_POSIX #define SANITIZER_INTERCEPT_TIME SI_POSIX +#define SANITIZER_INTERCEPT_TIMESPEC_GET SI_LINUX #define SANITIZER_INTERCEPT_GLOB (SI_GLIBC || SI_SOLARIS) #define SANITIZER_INTERCEPT_GLOB64 SI_GLIBC #define SANITIZER_INTERCEPT___B64_TO SI_LINUX_NOT_ANDROID @@ -274,8 +304,9 @@ #if SI_LINUX_NOT_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \ - defined(__s390__) || defined(__loongarch__) || SANITIZER_RISCV64) -#define SANITIZER_INTERCEPT_PTRACE 1 + defined(__s390__) || defined(__loongarch__) || SANITIZER_RISCV64 || \ + defined(__sparc__)) +# define SANITIZER_INTERCEPT_PTRACE 1 #else #define SANITIZER_INTERCEPT_PTRACE 0 #endif @@ -314,6 +345,8 @@ #define SANITIZER_INTERCEPT_GETGROUPS SI_POSIX #define SANITIZER_INTERCEPT_POLL SI_POSIX #define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID || SI_SOLARIS +#define SANITIZER_INTERCEPT_EPOLL (SI_LINUX) +#define SANITIZER_INTERCEPT_KQUEUE (SI_FREEBSD || SI_NETBSD || SI_MAC) #define SANITIZER_INTERCEPT_WORDEXP \ (SI_FREEBSD || SI_NETBSD || (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID || \ SI_SOLARIS) @@ -494,7 +527,8 @@ #define SANITIZER_INTERCEPT_PVALLOC (SI_GLIBC || SI_ANDROID) #define SANITIZER_INTERCEPT_CFREE (SI_GLIBC && !SANITIZER_RISCV64) #define SANITIZER_INTERCEPT_REALLOCARRAY SI_POSIX -#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC) +#define SANITIZER_INTERCEPT_ALIGNED_ALLOC \ + (!SI_MAC || SI_MAC_SDK_10_15_AVAILABLE) #define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD) #define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_WCSLEN 1 @@ -559,10 +593,8 @@ #define SANITIZER_INTERCEPT_SHA1 SI_NETBSD #define SANITIZER_INTERCEPT_MD4 SI_NETBSD #define SANITIZER_INTERCEPT_RMD160 SI_NETBSD -#define SANITIZER_INTERCEPT_MD5 (SI_NETBSD || SI_FREEBSD) -#define SANITIZER_INTERCEPT_FSEEK (SI_NETBSD || SI_FREEBSD) +#define SANITIZER_INTERCEPT_FSEEK SI_POSIX #define SANITIZER_INTERCEPT_MD2 SI_NETBSD -#define SANITIZER_INTERCEPT_SHA2 (SI_NETBSD || SI_FREEBSD) #define SANITIZER_INTERCEPT_CDB SI_NETBSD #define SANITIZER_INTERCEPT_VIS (SI_NETBSD || SI_FREEBSD) #define SANITIZER_INTERCEPT_POPEN SI_POSIX @@ -601,7 +633,13 @@ // FIXME: also available from musl 1.2.5 #define SANITIZER_INTERCEPT_PREADV2 (SI_LINUX && __GLIBC_PREREQ(2, 26)) #define SANITIZER_INTERCEPT_PWRITEV2 (SI_LINUX && __GLIBC_PREREQ(2, 26)) - +#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && \ + __MAC_OS_X_VERSION_MIN_REQUIRED >= 130000 +# define SI_MAC_OS_DEPLOYMENT_MIN_13_00 1 +#else +# define SI_MAC_OS_DEPLOYMENT_MIN_13_00 0 +#endif +#define SANITIZER_INTERCEPT_FREADLINK (SI_MAC && SI_MAC_OS_DEPLOYMENT_MIN_13_00) // This macro gives a way for downstream users to override the above // interceptor macros irrespective of the platform they are on. They have // to do two things: diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.cpp b/lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.cpp @@ -547,6 +547,7 @@ unsigned pid_t_sz = sizeof(pid_t); unsigned timeval_sz = sizeof(timeval); unsigned uid_t_sz = sizeof(uid_t); unsigned gid_t_sz = sizeof(gid_t); +unsigned fpos_t_sz = sizeof(fpos_t); unsigned mbstate_t_sz = sizeof(mbstate_t); unsigned sigset_t_sz = sizeof(sigset_t); unsigned struct_timezone_sz = sizeof(struct timezone); @@ -2487,8 +2488,6 @@ const unsigned RMD160_return_length = RMD160_DIGEST_STRING_LENGTH; const unsigned MD5_CTX_sz = sizeof(MD5_CTX); const unsigned MD5_return_length = MD5_DIGEST_STRING_LENGTH; -const unsigned fpos_t_sz = sizeof(fpos_t); - const unsigned MD2_CTX_sz = sizeof(MD2_CTX); const unsigned MD2_return_length = MD2_DIGEST_STRING_LENGTH; diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.h b/lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.h @@ -36,6 +36,7 @@ extern unsigned pid_t_sz; extern unsigned timeval_sz; extern unsigned uid_t_sz; extern unsigned gid_t_sz; +extern unsigned fpos_t_sz; extern unsigned mbstate_t_sz; extern unsigned struct_timezone_sz; extern unsigned struct_tms_sz; @@ -2335,8 +2336,6 @@ extern const unsigned RMD160_return_length; extern const unsigned MD5_CTX_sz; extern const unsigned MD5_return_length; -extern const unsigned fpos_t_sz; - extern const unsigned MD2_CTX_sz; extern const unsigned MD2_return_length; diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.cpp b/lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.cpp @@ -94,8 +94,9 @@ #if SANITIZER_LINUX # include <utime.h> # include <sys/ptrace.h> -# if defined(__mips64) || defined(__aarch64__) || defined(__arm__) || \ - defined(__hexagon__) || defined(__loongarch__) ||SANITIZER_RISCV64 +# if defined(__mips64) || defined(__aarch64__) || defined(__arm__) || \ + defined(__hexagon__) || defined(__loongarch__) || SANITIZER_RISCV64 || \ + defined(__sparc__) # include <asm/ptrace.h> # ifdef __arm__ typedef struct user_fpregs elf_fpregset_t; @@ -117,15 +118,16 @@ typedef struct user_fpregs elf_fpregset_t; #if SANITIZER_LINUX #if SANITIZER_GLIBC #include <fstab.h> -#include <net/if_ppp.h> -#include <netax25/ax25.h> -#include <netipx/ipx.h> -#include <netrom/netrom.h> -#include <obstack.h> -#if HAVE_RPC_XDR_H -# include <rpc/xdr.h> -#endif -#include <scsi/scsi.h> +# include <linux/filter.h> +# include <net/if_ppp.h> +# include <netax25/ax25.h> +# include <netipx/ipx.h> +# include <netrom/netrom.h> +# include <obstack.h> +# if HAVE_RPC_XDR_H +# include <rpc/xdr.h> +# endif +# include <scsi/scsi.h> #else #include <linux/if_ppp.h> #include <linux/kd.h> @@ -358,11 +360,12 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); const int wordexp_wrde_dooffs = WRDE_DOOFFS; # endif // !SANITIZER_ANDROID -#if SANITIZER_LINUX && !SANITIZER_ANDROID && \ - (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ - defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \ - defined(__s390__) || defined(__loongarch__)|| SANITIZER_RISCV64) -#if defined(__mips64) || defined(__powerpc64__) || defined(__arm__) +# if SANITIZER_LINUX && !SANITIZER_ANDROID && \ + (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ + defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \ + defined(__s390__) || defined(__loongarch__) || SANITIZER_RISCV64 || \ + defined(__sparc__)) +# if defined(__mips64) || defined(__powerpc64__) || defined(__arm__) unsigned struct_user_regs_struct_sz = sizeof(struct pt_regs); unsigned struct_user_fpregs_struct_sz = sizeof(elf_fpregset_t); #elif SANITIZER_RISCV64 @@ -377,19 +380,22 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); #elif defined(__s390__) unsigned struct_user_regs_struct_sz = sizeof(struct _user_regs_struct); unsigned struct_user_fpregs_struct_sz = sizeof(struct _user_fpregs_struct); -#else +# elif defined(__sparc__) + unsigned struct_user_regs_struct_sz = sizeof(struct sunos_regs); + unsigned struct_user_fpregs_struct_sz = sizeof(struct sunos_fp); +# else unsigned struct_user_regs_struct_sz = sizeof(struct user_regs_struct); unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpregs_struct); -#endif // __mips64 || __powerpc64__ || __aarch64__ || __loongarch__ -#if defined(__x86_64) || defined(__mips64) || defined(__powerpc64__) || \ - defined(__aarch64__) || defined(__arm__) || defined(__s390__) || \ - defined(__loongarch__) || SANITIZER_RISCV64 +# endif // __mips64 || __powerpc64__ || __aarch64__ || __loongarch__ +# if defined(__x86_64) || defined(__mips64) || defined(__powerpc64__) || \ + defined(__aarch64__) || defined(__arm__) || defined(__s390__) || \ + defined(__loongarch__) || SANITIZER_RISCV64 || defined(__sparc__) unsigned struct_user_fpxregs_struct_sz = 0; #else unsigned struct_user_fpxregs_struct_sz = sizeof(struct user_fpxregs_struct); #endif // __x86_64 || __mips64 || __powerpc64__ || __aarch64__ || __arm__ -// || __s390__ || __loongarch__ -#ifdef __arm__ + // || __s390__ || __loongarch__ || SANITIZER_RISCV64 || __sparc__ +# ifdef __arm__ unsigned struct_user_vfpregs_struct_sz = ARM_VFPREGS_SIZE; #else unsigned struct_user_vfpregs_struct_sz = 0; @@ -531,13 +537,16 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info); unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats); -#endif // SANITIZER_GLIBC + unsigned struct_sock_fprog_sz = sizeof(struct sock_fprog); +# endif // SANITIZER_GLIBC -#if !SANITIZER_ANDROID && !SANITIZER_APPLE +# if !SANITIZER_ANDROID && !SANITIZER_APPLE unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req); unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req); #endif + unsigned fpos_t_sz = sizeof(fpos_t); + const unsigned long __sanitizer_bufsiz = BUFSIZ; const unsigned IOCTL_NOT_PRESENT = 0; @@ -1084,7 +1093,7 @@ CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len); CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level); CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type); -#if SANITIZER_LINUX && (__ANDROID_API__ >= 21 || __GLIBC_PREREQ (2, 14)) +# if SANITIZER_LINUX && (SANITIZER_ANDROID || __GLIBC_PREREQ(2, 14)) CHECK_TYPE_SIZE(mmsghdr); CHECK_SIZE_AND_OFFSET(mmsghdr, msg_hdr); CHECK_SIZE_AND_OFFSET(mmsghdr, msg_len); diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.h b/lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.h @@ -98,10 +98,13 @@ const unsigned struct_kernel_stat64_sz = 104; const unsigned struct_kernel_stat_sz = 144; const unsigned struct_kernel_stat64_sz = 104; #elif defined(__mips__) -const unsigned struct_kernel_stat_sz = - SANITIZER_ANDROID - ? FIRST_32_SECOND_64(104, 128) - : FIRST_32_SECOND_64((_MIPS_SIM == _ABIN32) ? 176 : 160, 216); +const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID + ? FIRST_32_SECOND_64(104, 128) +# if defined(_ABIN32) && _MIPS_SIM == _ABIN32 + : FIRST_32_SECOND_64(176, 216); +# else + : FIRST_32_SECOND_64(160, 216); +# endif const unsigned struct_kernel_stat64_sz = 104; #elif defined(__s390__) && !defined(__s390x__) const unsigned struct_kernel_stat_sz = 64; @@ -313,7 +316,7 @@ extern unsigned struct_statvfs_sz; struct __sanitizer_iovec { void *iov_base; - uptr iov_len; + usize iov_len; }; #if !SANITIZER_ANDROID @@ -389,6 +392,16 @@ typedef long __sanitizer_time_t; typedef long __sanitizer_suseconds_t; +struct __sanitizer_timespec { + __sanitizer_time_t tv_sec; /* seconds */ + u64 tv_nsec; /* nanoseconds */ +}; + +struct __sanitizer_itimerspec { + struct __sanitizer_timespec it_interval; /* timer period */ + struct __sanitizer_timespec it_value; /* timer expiration */ +}; + struct __sanitizer_timeval { __sanitizer_time_t tv_sec; __sanitizer_suseconds_t tv_usec; @@ -513,6 +526,7 @@ struct __sanitizer_dirent64 { unsigned short d_reclen; // more fields that we don't care about }; +extern unsigned struct_sock_fprog_sz; #endif #if defined(__x86_64__) && !defined(_LP64) @@ -590,7 +604,7 @@ struct __sanitizer_siginfo_pad { #if SANITIZER_LINUX # define SANITIZER_HAS_SIGINFO 1 union __sanitizer_siginfo { - struct { + __extension__ struct { int si_signo; # if SANITIZER_MIPS int si_code; @@ -855,10 +869,11 @@ typedef void __sanitizer_FILE; # define SANITIZER_HAS_STRUCT_FILE 0 #endif -#if SANITIZER_LINUX && !SANITIZER_ANDROID && \ - (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ - defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \ - defined(__s390__) || defined(__loongarch__) || SANITIZER_RISCV64) +# if SANITIZER_LINUX && !SANITIZER_ANDROID && \ + (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ + defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \ + defined(__s390__) || defined(__loongarch__) || SANITIZER_RISCV64 || \ + defined(__sparc__)) extern unsigned struct_user_regs_struct_sz; extern unsigned struct_user_fpregs_struct_sz; extern unsigned struct_user_fpxregs_struct_sz; @@ -880,9 +895,24 @@ extern int ptrace_setsiginfo; extern int ptrace_getregset; extern int ptrace_setregset; extern int ptrace_geteventmsg; -#endif -#if SANITIZER_LINUX && !SANITIZER_ANDROID +// Helper for the ptrace interceptor. +template <class T> +inline T ptrace_data_arg(int request, T addr, T data) { +# if SANITIZER_LINUX && SANITIZER_SPARC + // As described in ptrace(2), the meanings of addr and data are reversed + // for the PTRACE_GETREGS, PTRACE_GETFPREGS, PTRACE_GETREGS, and + // PTRACE_GETFPREGS requests on Linux/sparc64. + if (request == ptrace_getregs || request == ptrace_getfpregs || + request == ptrace_setregs || request == ptrace_setfpregs) + return addr; + else +# endif + return data; +} +# endif + +# if SANITIZER_LINUX && !SANITIZER_ANDROID extern unsigned struct_shminfo_sz; extern unsigned struct_shm_info_sz; extern int shmctl_ipc_stat; @@ -1050,7 +1080,7 @@ extern unsigned struct_serial_struct_sz; extern unsigned struct_sockaddr_ax25_sz; extern unsigned struct_unimapdesc_sz; extern unsigned struct_unimapinit_sz; -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID +# endif // SANITIZER_LINUX && !SANITIZER_ANDROID extern const unsigned long __sanitizer_bufsiz; @@ -1064,6 +1094,8 @@ extern unsigned struct_sioc_sg_req_sz; extern unsigned struct_sioc_vif_req_sz; #endif +extern unsigned fpos_t_sz; + // ioctl request identifiers // A special value to mark ioctls that are not present on the target platform, @@ -1500,6 +1532,10 @@ extern const int si_SEGV_ACCERR; #define SIGACTION_SYMNAME sigaction +# if SANITIZER_LINUX +typedef void *__sanitizer_timer_t; +# endif + #endif // SANITIZER_LINUX || SANITIZER_APPLE #endif diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.cpp b/lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.cpp @@ -32,6 +32,7 @@ #include <semaphore.h> #include <signal.h> #include <stddef.h> +#include <stdio.h> #include <sys/ethernet.h> #include <sys/filio.h> #include <sys/ipc.h> @@ -135,6 +136,8 @@ namespace __sanitizer { unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req); unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req); + unsigned fpos_t_sz = sizeof(fpos_t); + const unsigned IOCTL_NOT_PRESENT = 0; unsigned IOCTL_FIOASYNC = FIOASYNC; diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.h b/lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.h @@ -418,6 +418,8 @@ extern unsigned struct_winsize_sz; extern unsigned struct_sioc_sg_req_sz; extern unsigned struct_sioc_vif_req_sz; +extern unsigned fpos_t_sz; + // ioctl request identifiers // A special value to mark ioctls that are not present on the target platform, diff --git a/lib/tsan/sanitizer_common/sanitizer_posix.cpp b/lib/tsan/sanitizer_common/sanitizer_posix.cpp @@ -353,7 +353,15 @@ bool ShouldMockFailureToOpen(const char *path) { internal_strncmp(path, "/proc/", 6) == 0; } -#if SANITIZER_LINUX && !SANITIZER_ANDROID && !SANITIZER_GO +bool OpenReadsVaArgs(int oflag) { +# ifdef O_TMPFILE + return (oflag & (O_CREAT | O_TMPFILE)) != 0; +# else + return (oflag & O_CREAT) != 0; +# endif +} + +# if SANITIZER_LINUX && !SANITIZER_ANDROID && !SANITIZER_GO int GetNamedMappingFd(const char *name, uptr size, int *flags) { if (!common_flags()->decorate_proc_maps || !name) return -1; diff --git a/lib/tsan/sanitizer_common/sanitizer_posix.h b/lib/tsan/sanitizer_common/sanitizer_posix.h @@ -28,6 +28,9 @@ namespace __sanitizer { // Don't use directly, use __sanitizer::OpenFile() instead. uptr internal_open(const char *filename, int flags); uptr internal_open(const char *filename, int flags, u32 mode); +# if SANITIZER_FREEBSD +uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags); +# endif uptr internal_close(fd_t fd); uptr internal_read(fd_t fd, void *buf, uptr count); @@ -86,7 +89,7 @@ int internal_pthread_join(void *th, void **ret); return REAL(pthread_create)(th, attr, callback, param); \ } \ int internal_pthread_join(void *th, void **ret) { \ - return REAL(pthread_join(th, ret)); \ + return REAL(pthread_join)(th, ret); \ } \ } // namespace __sanitizer @@ -108,6 +111,7 @@ bool IsStateDetached(int state); fd_t ReserveStandardFds(fd_t fd); bool ShouldMockFailureToOpen(const char *path); +bool OpenReadsVaArgs(int oflag); // Create a non-file mapping with a given /proc/self/maps name. uptr MmapNamed(void *addr, uptr length, int prot, int flags, const char *name); diff --git a/lib/tsan/sanitizer_common/sanitizer_posix_libcdep.cpp b/lib/tsan/sanitizer_common/sanitizer_posix_libcdep.cpp @@ -288,26 +288,86 @@ bool SignalContext::IsStackOverflow() const { #endif // SANITIZER_GO +static void SetNonBlock(int fd) { + int res = fcntl(fd, F_GETFL, 0); + CHECK(!internal_iserror(res, nullptr)); + + res |= O_NONBLOCK; + res = fcntl(fd, F_SETFL, res); + CHECK(!internal_iserror(res, nullptr)); +} + bool IsAccessibleMemoryRange(uptr beg, uptr size) { - uptr page_size = GetPageSizeCached(); - // Checking too large memory ranges is slow. - CHECK_LT(size, page_size * 10); - int sock_pair[2]; - if (pipe(sock_pair)) - return false; - uptr bytes_written = - internal_write(sock_pair[1], reinterpret_cast<void *>(beg), size); - int write_errno; - bool result; - if (internal_iserror(bytes_written, &write_errno)) { - CHECK_EQ(EFAULT, write_errno); - result = false; - } else { - result = (bytes_written == size); + while (size) { + // `read` from `fds[0]` into a dummy buffer to free up the pipe buffer for + // more `write` is slower than just recreating a pipe. + int fds[2]; + CHECK_EQ(0, pipe(fds)); + + auto cleanup = at_scope_exit([&]() { + internal_close(fds[0]); + internal_close(fds[1]); + }); + + SetNonBlock(fds[1]); + + int write_errno; + uptr w = internal_write(fds[1], reinterpret_cast<char *>(beg), size); + if (internal_iserror(w, &write_errno)) { + if (write_errno == EINTR) + continue; + CHECK_EQ(EFAULT, write_errno); + return false; + } + size -= w; + beg += w; + } + + return true; +} + +bool TryMemCpy(void *dest, const void *src, uptr n) { + if (!n) + return true; + int fds[2]; + CHECK_EQ(0, pipe(fds)); + + auto cleanup = at_scope_exit([&]() { + internal_close(fds[0]); + internal_close(fds[1]); + }); + + SetNonBlock(fds[0]); + SetNonBlock(fds[1]); + + char *d = static_cast<char *>(dest); + const char *s = static_cast<const char *>(src); + + while (n) { + int e; + uptr w = internal_write(fds[1], s, n); + if (internal_iserror(w, &e)) { + if (e == EINTR) + continue; + CHECK_EQ(EFAULT, e); + return false; + } + s += w; + n -= w; + + while (w) { + uptr r = internal_read(fds[0], d, w); + if (internal_iserror(r, &e)) { + CHECK_EQ(EINTR, e); + continue; + } + + d += r; + w -= r; + } } - internal_close(sock_pair[0]); - internal_close(sock_pair[1]); - return result; + + return true; } void PlatformPrepareForSandboxing(void *args) { @@ -483,7 +543,11 @@ pid_t StartSubprocess(const char *program, const char *const argv[], internal_close(stderr_fd); } +# if SANITIZER_FREEBSD + internal_close_range(3, ~static_cast<fd_t>(0), 0); +# else for (int fd = sysconf(_SC_OPEN_MAX); fd > 2; fd--) internal_close(fd); +# endif internal_execve(program, const_cast<char **>(&argv[0]), const_cast<char *const *>(envp)); diff --git a/lib/tsan/sanitizer_common/sanitizer_procmaps_mac.cpp b/lib/tsan/sanitizer_common/sanitizer_procmaps_mac.cpp @@ -433,7 +433,9 @@ void MemoryMappingLayout::DumpListOfModules( MemoryMappedSegmentData data; segment.data_ = &data; while (Next(&segment)) { - if (segment.filename[0] == '\0') continue; + // skip the __PAGEZERO segment, its vmsize is 0 + if (segment.filename[0] == '\0' || (segment.start == segment.end)) + continue; LoadedModule *cur_module = nullptr; if (!modules->empty() && 0 == internal_strcmp(segment.filename, modules->back().full_name())) { diff --git a/lib/tsan/sanitizer_common/sanitizer_procmaps_solaris.cpp b/lib/tsan/sanitizer_common/sanitizer_procmaps_solaris.cpp @@ -11,6 +11,10 @@ // Before Solaris 11.4, <procfs.h> doesn't work in a largefile environment. #undef _FILE_OFFSET_BITS + +// Avoid conflict between `_TIME_BITS` defined vs. `_FILE_OFFSET_BITS` +// undefined in some Linux configurations. +#undef _TIME_BITS #include "sanitizer_platform.h" #if SANITIZER_SOLARIS # include <fcntl.h> diff --git a/lib/tsan/sanitizer_common/sanitizer_redefine_builtins.h b/lib/tsan/sanitizer_common/sanitizer_redefine_builtins.h @@ -17,9 +17,11 @@ // The asm hack only works with GCC and Clang. # if !defined(_WIN32) -asm("memcpy = __sanitizer_internal_memcpy"); -asm("memmove = __sanitizer_internal_memmove"); -asm("memset = __sanitizer_internal_memset"); +asm(R"( + .set memcpy, __sanitizer_internal_memcpy + .set memmove, __sanitizer_internal_memmove + .set memset, __sanitizer_internal_memset + )"); # if defined(__cplusplus) && \ !defined(SANITIZER_COMMON_REDEFINE_BUILTINS_IN_STD) diff --git a/lib/tsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp b/lib/tsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp @@ -137,10 +137,6 @@ class ThreadSuspender { }; bool ThreadSuspender::SuspendThread(tid_t tid) { - // Are we already attached to this thread? - // Currently this check takes linear time, however the number of threads is - // usually small. - if (suspended_threads_list_.ContainsTid(tid)) return false; int pterrno; if (internal_iserror(internal_ptrace(PTRACE_ATTACH, tid, nullptr, nullptr), &pterrno)) { @@ -217,17 +213,28 @@ bool ThreadSuspender::SuspendAllThreads() { switch (thread_lister.ListThreads(&threads)) { case ThreadLister::Error: ResumeAllThreads(); + VReport(1, "Failed to list threads\n"); return false; case ThreadLister::Incomplete: + VReport(1, "Incomplete list\n"); retry = true; break; case ThreadLister::Ok: break; } for (tid_t tid : threads) { + // Are we already attached to this thread? + // Currently this check takes linear time, however the number of threads + // is usually small. + if (suspended_threads_list_.ContainsTid(tid)) + continue; if (SuspendThread(tid)) retry = true; + else + VReport(2, "%llu/status: %s\n", tid, thread_lister.LoadStatus(tid)); } + if (retry) + VReport(1, "SuspendAllThreads retry: %d\n", i); } return suspended_threads_list_.ThreadCount(); } diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer.h b/lib/tsan/sanitizer_common/sanitizer_symbolizer.h @@ -185,17 +185,17 @@ class Symbolizer final { class ModuleNameOwner { public: explicit ModuleNameOwner(Mutex *synchronized_by) - : last_match_(nullptr), mu_(synchronized_by) { + : mu_(synchronized_by), last_match_(nullptr) { storage_.reserve(kInitialCapacity); } const char *GetOwnedCopy(const char *str); private: static const uptr kInitialCapacity = 1000; - InternalMmapVector<const char*> storage_; - const char *last_match_; Mutex *mu_; + const char *last_match_ SANITIZER_GUARDED_BY(mu_); + InternalMmapVector<const char *> storage_ SANITIZER_GUARDED_BY(*mu_); } module_names_; /// Platform-specific function for creating a Symbolizer object. @@ -220,7 +220,7 @@ class Symbolizer final { // always synchronized. Mutex mu_; - IntrusiveList<SymbolizerTool> tools_; + IntrusiveList<SymbolizerTool> tools_ SANITIZER_GUARDED_BY(mu_); explicit Symbolizer(IntrusiveList<SymbolizerTool> tools); diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.cpp b/lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.cpp @@ -30,7 +30,7 @@ namespace __sanitizer { bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) { Dl_info info; int result = dladdr((const void *)addr, &info); - if (!result) return false; + if (!result || !info.dli_sname) return false; // Compute offset if possible. `dladdr()` doesn't always ensure that `addr >= // sym_addr` so only compute the offset when this holds. Failure to find the @@ -51,7 +51,7 @@ bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) { bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) { Dl_info info; int result = dladdr((const void *)addr, &info); - if (!result) return false; + if (!result || !info.dli_sname) return false; const char *demangled = DemangleSwiftAndCXX(info.dli_sname); if (!demangled) demangled = info.dli_sname; diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer_report.cpp b/lib/tsan/sanitizer_common/sanitizer_symbolizer_report.cpp @@ -227,12 +227,15 @@ static void ReportStackOverflowImpl(const SignalContext &sig, u32 tid, SanitizerToolName, kDescription, (void *)sig.addr, (void *)sig.pc, (void *)sig.bp, (void *)sig.sp, tid); Printf("%s", d.Default()); - InternalMmapVector<BufferedStackTrace> stack_buffer(1); - BufferedStackTrace *stack = stack_buffer.data(); - stack->Reset(); - unwind(sig, unwind_context, stack); - stack->Print(); - ReportErrorSummary(kDescription, stack); + // Avoid SEGVs in the unwinder when bp couldn't be determined. + if (sig.bp) { + InternalMmapVector<BufferedStackTrace> stack_buffer(1); + BufferedStackTrace *stack = stack_buffer.data(); + stack->Reset(); + unwind(sig, unwind_context, stack); + stack->Print(); + ReportErrorSummary(kDescription, stack); + } } static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid, diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer_win.cpp b/lib/tsan/sanitizer_common/sanitizer_symbolizer_win.cpp @@ -65,12 +65,13 @@ void InitializeDbgHelpIfNeeded() { HMODULE dbghelp = LoadLibraryA("dbghelp.dll"); CHECK(dbghelp && "failed to load dbghelp.dll"); -#define DBGHELP_IMPORT(name) \ - do { \ - name = \ - reinterpret_cast<decltype(::name) *>(GetProcAddress(dbghelp, #name)); \ - CHECK(name != nullptr); \ - } while (0) +# define DBGHELP_IMPORT(name) \ + do { \ + name = reinterpret_cast<decltype(::name) *>( \ + (void *)GetProcAddress(dbghelp, #name)); \ + CHECK(name != nullptr); \ + } while (0) + DBGHELP_IMPORT(StackWalk64); DBGHELP_IMPORT(SymCleanup); DBGHELP_IMPORT(SymFromAddr); diff --git a/lib/tsan/sanitizer_common/sanitizer_thread_history.cpp b/lib/tsan/sanitizer_common/sanitizer_thread_history.cpp @@ -0,0 +1,72 @@ +//===-- sanitizer_thread_history.cpp --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_thread_history.h" + +#include "sanitizer_stackdepot.h" +namespace __sanitizer { + +void PrintThreadHistory(ThreadRegistry ®istry, InternalScopedString &out) { + ThreadRegistryLock l(®istry); + // Stack traces are largest part of printout and they often the same for + // multiple threads, so we will deduplicate them. + InternalMmapVector<const ThreadContextBase *> stacks; + + registry.RunCallbackForEachThreadLocked( + [](ThreadContextBase *context, void *arg) { + static_cast<decltype(&stacks)>(arg)->push_back(context); + }, + &stacks); + + Sort(stacks.data(), stacks.size(), + [](const ThreadContextBase *a, const ThreadContextBase *b) { + if (a->stack_id < b->stack_id) + return true; + if (a->stack_id > b->stack_id) + return false; + return a->unique_id < b->unique_id; + }); + + auto describe_thread = [&](const ThreadContextBase *context) { + if (!context) { + out.Append("T-1"); + return; + } + out.AppendF("T%llu/%llu", context->unique_id, context->os_id); + if (internal_strlen(context->name)) + out.AppendF(" (%s)", context->name); + }; + + auto get_parent = + [&](const ThreadContextBase *context) -> const ThreadContextBase * { + if (!context) + return nullptr; + ThreadContextBase *parent = registry.GetThreadLocked(context->parent_tid); + if (!parent) + return nullptr; + if (parent->unique_id >= context->unique_id) + return nullptr; + return parent; + }; + + const ThreadContextBase *prev = nullptr; + for (const ThreadContextBase *context : stacks) { + if (prev && prev->stack_id != context->stack_id) + StackDepotGet(prev->stack_id).PrintTo(&out); + prev = context; + out.Append("Thread "); + describe_thread(context); + out.Append(" was created by "); + describe_thread(get_parent(context)); + out.Append("\n"); + } + if (prev) + StackDepotGet(prev->stack_id).PrintTo(&out); +} + +} // namespace __sanitizer diff --git a/lib/tsan/sanitizer_common/sanitizer_thread_history.h b/lib/tsan/sanitizer_common/sanitizer_thread_history.h @@ -0,0 +1,24 @@ +//===-- sanitizer_thread_history.h ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utility to print thread histroy from ThreadRegistry. +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_THREAD_HISTORY_H +#define SANITIZER_THREAD_HISTORY_H + +#include "sanitizer_thread_registry.h" + +namespace __sanitizer { + +void PrintThreadHistory(ThreadRegistry& registry, InternalScopedString& out); + +} // namespace __sanitizer + +#endif // SANITIZER_THREAD_HISTORY_H diff --git a/lib/tsan/sanitizer_common/sanitizer_thread_registry.cpp b/lib/tsan/sanitizer_common/sanitizer_thread_registry.cpp @@ -18,9 +18,17 @@ namespace __sanitizer { ThreadContextBase::ThreadContextBase(u32 tid) - : tid(tid), unique_id(0), reuse_count(), os_id(0), user_id(0), - status(ThreadStatusInvalid), detached(false), - thread_type(ThreadType::Regular), parent_tid(0), next(0) { + : tid(tid), + unique_id(0), + reuse_count(), + os_id(0), + user_id(0), + status(ThreadStatusInvalid), + detached(false), + thread_type(ThreadType::Regular), + parent_tid(0), + stack_id(0), + next(0) { name[0] = '\0'; atomic_store(&thread_destroyed, 0, memory_order_release); } @@ -39,8 +47,7 @@ void ThreadContextBase::SetName(const char *new_name) { } void ThreadContextBase::SetDead() { - CHECK(status == ThreadStatusRunning || - status == ThreadStatusFinished); + CHECK(status == ThreadStatusRunning || status == ThreadStatusFinished); status = ThreadStatusDead; user_id = 0; OnDead(); @@ -68,7 +75,8 @@ void ThreadContextBase::SetFinished() { // for a thread that never actually started. In that case the thread // should go to ThreadStatusFinished regardless of whether it was created // as detached. - if (!detached || status == ThreadStatusCreated) status = ThreadStatusFinished; + if (!detached || status == ThreadStatusCreated) + status = ThreadStatusFinished; OnFinished(); } @@ -81,14 +89,17 @@ void ThreadContextBase::SetStarted(tid_t _os_id, ThreadType _thread_type, } void ThreadContextBase::SetCreated(uptr _user_id, u64 _unique_id, - bool _detached, u32 _parent_tid, void *arg) { + bool _detached, u32 _parent_tid, + u32 _stack_tid, void *arg) { status = ThreadStatusCreated; user_id = _user_id; unique_id = _unique_id; detached = _detached; // Parent tid makes no sense for the main thread. - if (tid != kMainTid) + if (tid != kMainTid) { parent_tid = _parent_tid; + stack_id = _stack_tid; + } OnCreated(arg); } @@ -124,8 +135,10 @@ void ThreadRegistry::GetNumberOfThreads(uptr *total, uptr *running, ThreadRegistryLock l(this); if (total) *total = threads_.size(); - if (running) *running = running_threads_; - if (alive) *alive = alive_threads_; + if (running) + *running = running_threads_; + if (alive) + *alive = alive_threads_; } uptr ThreadRegistry::GetMaxAliveThreads() { @@ -134,7 +147,7 @@ uptr ThreadRegistry::GetMaxAliveThreads() { } u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid, - void *arg) { + u32 stack_tid, void *arg) { ThreadRegistryLock l(this); u32 tid = kInvalidTid; ThreadContextBase *tctx = QuarantinePop(); @@ -150,8 +163,10 @@ u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid, Report("%s: Thread limit (%u threads) exceeded. Dying.\n", SanitizerToolName, max_threads_); #else - Printf("race: limit on %u simultaneously alive goroutines is exceeded," - " dying\n", max_threads_); + Printf( + "race: limit on %u simultaneously alive goroutines is exceeded," + " dying\n", + max_threads_); #endif Die(); } @@ -170,8 +185,8 @@ u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid, // positives later (e.g. if we join a wrong thread). CHECK(live_.try_emplace(user_id, tid).second); } - tctx->SetCreated(user_id, total_threads_++, detached, - parent_tid, arg); + tctx->SetCreated(user_id, total_threads_++, detached, parent_tid, stack_tid, + arg); return tid; } @@ -196,8 +211,8 @@ u32 ThreadRegistry::FindThread(FindThreadCallback cb, void *arg) { return kInvalidTid; } -ThreadContextBase * -ThreadRegistry::FindThreadContextLocked(FindThreadCallback cb, void *arg) { +ThreadContextBase *ThreadRegistry::FindThreadContextLocked( + FindThreadCallback cb, void *arg) { CheckLocked(); for (u32 tid = 0; tid < threads_.size(); tid++) { ThreadContextBase *tctx = threads_[tid]; @@ -210,7 +225,7 @@ ThreadRegistry::FindThreadContextLocked(FindThreadCallback cb, void *arg) { static bool FindThreadContextByOsIdCallback(ThreadContextBase *tctx, void *arg) { return (tctx->os_id == (uptr)arg && tctx->status != ThreadStatusInvalid && - tctx->status != ThreadStatusDead); + tctx->status != ThreadStatusDead); } ThreadContextBase *ThreadRegistry::FindThreadContextByOsIDLocked(tid_t os_id) { diff --git a/lib/tsan/sanitizer_common/sanitizer_thread_registry.h b/lib/tsan/sanitizer_common/sanitizer_thread_registry.h @@ -52,6 +52,7 @@ class ThreadContextBase { ThreadType thread_type; u32 parent_tid; + u32 stack_id; ThreadContextBase *next; // For storing thread contexts in a list. atomic_uint32_t thread_destroyed; // To address race of Joined vs Finished @@ -63,7 +64,7 @@ class ThreadContextBase { void SetFinished(); void SetStarted(tid_t _os_id, ThreadType _thread_type, void *arg); void SetCreated(uptr _user_id, u64 _unique_id, bool _detached, - u32 _parent_tid, void *arg); + u32 _parent_tid, u32 _stack_tid, void *arg); void Reset(); void SetDestroyed(); @@ -101,12 +102,16 @@ class SANITIZER_MUTEX ThreadRegistry { // Should be guarded by ThreadRegistryLock. ThreadContextBase *GetThreadLocked(u32 tid) { - return threads_.empty() ? nullptr : threads_[tid]; + return tid < threads_.size() ? threads_[tid] : nullptr; } u32 NumThreadsLocked() const { return threads_.size(); } - u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg); + u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, u32 stack_tid, + void *arg); + u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg) { + return CreateThread(user_id, detached, parent_tid, 0, arg); + } typedef void (*ThreadCallback)(ThreadContextBase *tctx, void *arg); // Invokes callback with a specified arg for each thread context. diff --git a/lib/tsan/sanitizer_common/sanitizer_tls_get_addr.cpp b/lib/tsan/sanitizer_common/sanitizer_tls_get_addr.cpp @@ -14,6 +14,8 @@ #include "sanitizer_allocator_interface.h" #include "sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_flags.h" #include "sanitizer_platform_interceptors.h" @@ -66,7 +68,7 @@ static DTLS::DTVBlock *DTLS_NextBlock(atomic_uintptr_t *cur) { } static DTLS::DTV *DTLS_Find(uptr id) { - VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", (void *)&dtls, id); + VReport(3, "__tls_get_addr: DTLS_Find %p %zd\n", (void *)&dtls, id); static constexpr uptr kPerBlock = ARRAY_SIZE(DTLS::DTVBlock::dtvs); DTLS::DTVBlock *cur = DTLS_NextBlock(&dtls.dtv_block); if (!cur) @@ -110,6 +112,21 @@ SANITIZER_WEAK_ATTRIBUTE const void *__sanitizer_get_allocated_begin(const void *p); } +SANITIZER_INTERFACE_WEAK_DEF(uptr, __sanitizer_get_dtls_size, + const void *tls_begin) { + const void *start = __sanitizer_get_allocated_begin(tls_begin); + if (!start) + return 0; + CHECK_LE(start, tls_begin); + uptr tls_size = __sanitizer_get_allocated_size(start); + VReport(2, "__tls_get_addr: glibc DTLS suspected; tls={%p,0x%zx}\n", + tls_begin, tls_size); + uptr offset = + (reinterpret_cast<uptr>(tls_begin) - reinterpret_cast<uptr>(start)); + CHECK_LE(offset, tls_size); + return tls_size - offset; +} + DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res, uptr static_tls_begin, uptr static_tls_end) { if (!common_flags()->intercept_tls_get_addr) return 0; @@ -117,8 +134,8 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res, uptr dso_id = arg->dso_id; DTLS::DTV *dtv = DTLS_Find(dso_id); if (!dtv || dtv->beg) - return 0; - uptr tls_size = 0; + return nullptr; + CHECK_LE(static_tls_begin, static_tls_end); uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset; VReport(2, "__tls_get_addr: %p {0x%zx,0x%zx} => %p; tls_beg: %p; sp: %p " @@ -126,36 +143,33 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res, (void *)arg, arg->dso_id, arg->offset, res, (void *)tls_beg, (void *)&tls_beg, atomic_load(&number_of_live_dtls, memory_order_relaxed)); - if (dtls.last_memalign_ptr == tls_beg) { - tls_size = dtls.last_memalign_size; - VReport(2, "__tls_get_addr: glibc <=2.24 suspected; tls={%p,0x%zx}\n", - (void *)tls_beg, tls_size); - } else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) { + if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) { // This is the static TLS block which was initialized / unpoisoned at thread // creation. VReport(2, "__tls_get_addr: static tls: %p\n", (void *)tls_beg); - tls_size = 0; - } else if (const void *start = - __sanitizer_get_allocated_begin((void *)tls_beg)) { - tls_beg = (uptr)start; - tls_size = __sanitizer_get_allocated_size(start); - VReport(2, "__tls_get_addr: glibc >=2.25 suspected; tls={%p,0x%zx}\n", - (void *)tls_beg, tls_size); - } else { - VReport(2, "__tls_get_addr: Can't guess glibc version\n"); - // This may happen inside the DTOR of main thread, so just ignore it. - tls_size = 0; + dtv->beg = tls_beg; + dtv->size = 0; + return nullptr; } + if (uptr tls_size = + __sanitizer_get_dtls_size(reinterpret_cast<void *>(tls_beg))) { + dtv->beg = tls_beg; + dtv->size = tls_size; + return dtv; + } + VReport(2, "__tls_get_addr: Can't guess glibc version\n"); + // This may happen inside the DTOR a thread, or async signal handlers before + // thread initialization, so just ignore it. + // + // If the unknown block is dynamic TLS, unlikely we will be able to recognize + // it in future, mark it as done with '{tls_beg, 0}'. + // + // If the block is static TLS, possible reason of failed detection is nullptr + // in `static_tls_begin`. Regardless of reasons, the future handling of static + // TLS is still '{tls_beg, 0}'. dtv->beg = tls_beg; - dtv->size = tls_size; - return dtv; -} - -void DTLS_on_libc_memalign(void *ptr, uptr size) { - if (!common_flags()->intercept_tls_get_addr) return; - VReport(2, "DTLS_on_libc_memalign: %p 0x%zx\n", ptr, size); - dtls.last_memalign_ptr = reinterpret_cast<uptr>(ptr); - dtls.last_memalign_size = size; + dtv->size = 0; + return nullptr; } DTLS *DTLS_Get() { return &dtls; } @@ -166,7 +180,9 @@ bool DTLSInDestruction(DTLS *dtls) { } #else -void DTLS_on_libc_memalign(void *ptr, uptr size) {} +SANITIZER_INTERFACE_WEAK_DEF(uptr, __sanitizer_get_dtls_size, const void *) { + return 0; +} DTLS::DTV *DTLS_on_tls_get_addr(void *arg, void *res, unsigned long, unsigned long) { return 0; } DTLS *DTLS_Get() { return 0; } diff --git a/lib/tsan/sanitizer_common/sanitizer_tls_get_addr.h b/lib/tsan/sanitizer_common/sanitizer_tls_get_addr.h @@ -55,10 +55,6 @@ struct DTLS { static_assert(sizeof(DTVBlock) <= 4096UL, "Unexpected block size"); atomic_uintptr_t dtv_block; - - // Auxiliary fields, don't access them outside sanitizer_tls_get_addr.cpp - uptr last_memalign_size; - uptr last_memalign_ptr; }; template <typename Fn> diff --git a/lib/tsan/sanitizer_common/sanitizer_win.cpp b/lib/tsan/sanitizer_common/sanitizer_win.cpp @@ -164,7 +164,24 @@ void UnmapOrDie(void *addr, uptr size, bool raw_report) { static void *ReturnNullptrOnOOMOrDie(uptr size, const char *mem_type, const char *mmap_type) { error_t last_error = GetLastError(); - if (last_error == ERROR_NOT_ENOUGH_MEMORY) + + // Assumption: VirtualAlloc is the last system call that was invoked before + // this method. + // VirtualAlloc emits one of 3 error codes when running out of memory + // 1. ERROR_NOT_ENOUGH_MEMORY: + // There's not enough memory to execute the command + // 2. ERROR_INVALID_PARAMETER: + // VirtualAlloc will return this if the request would allocate memory at an + // address exceeding or being very close to the maximum application address + // (the `lpMaximumApplicationAddress` field within the `SystemInfo` struct). + // This does not seem to be officially documented, but is corroborated here: + // https://stackoverflow.com/questions/45833674/why-does-virtualalloc-fail-for-lpaddress-greater-than-0x6ffffffffff + // 3. ERROR_COMMITMENT_LIMIT: + // VirtualAlloc will return this if e.g. the pagefile is too small to commit + // the requested amount of memory. + if (last_error == ERROR_NOT_ENOUGH_MEMORY || + last_error == ERROR_INVALID_PARAMETER || + last_error == ERROR_COMMITMENT_LIMIT) return nullptr; ReportMmapFailureAndDie(size, mem_type, mmap_type, last_error); } @@ -873,24 +890,18 @@ uptr GetTlsSize() { return 0; } -void InitTlsSize() { -} - -void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, - uptr *tls_addr, uptr *tls_size) { -#if SANITIZER_GO - *stk_addr = 0; - *stk_size = 0; - *tls_addr = 0; - *tls_size = 0; -#else - uptr stack_top, stack_bottom; - GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom); - *stk_addr = stack_bottom; - *stk_size = stack_top - stack_bottom; - *tls_addr = 0; - *tls_size = 0; -#endif +void GetThreadStackAndTls(bool main, uptr *stk_begin, uptr *stk_end, + uptr *tls_begin, uptr *tls_end) { +# if SANITIZER_GO + *stk_begin = 0; + *stk_end = 0; + *tls_begin = 0; + *tls_end = 0; +# else + GetThreadStackTopAndBottom(main, stk_end, stk_begin); + *tls_begin = 0; + *tls_end = 0; +# endif } void ReportFile::Write(const char *buffer, uptr length) { @@ -974,6 +985,11 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) { return true; } +bool TryMemCpy(void *dest, const void *src, uptr n) { + // TODO: implement. + return false; +} + bool SignalContext::IsStackOverflow() const { return (DWORD)GetType() == EXCEPTION_STACK_OVERFLOW; } @@ -1039,7 +1055,52 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { } void SignalContext::DumpAllRegisters(void *context) { - // FIXME: Implement this. + CONTEXT *ctx = (CONTEXT *)context; +# if defined(_M_X64) + Report("Register values:\n"); + Printf("rax = %llx ", ctx->Rax); + Printf("rbx = %llx ", ctx->Rbx); + Printf("rcx = %llx ", ctx->Rcx); + Printf("rdx = %llx ", ctx->Rdx); + Printf("\n"); + Printf("rdi = %llx ", ctx->Rdi); + Printf("rsi = %llx ", ctx->Rsi); + Printf("rbp = %llx ", ctx->Rbp); + Printf("rsp = %llx ", ctx->Rsp); + Printf("\n"); + Printf("r8 = %llx ", ctx->R8); + Printf("r9 = %llx ", ctx->R9); + Printf("r10 = %llx ", ctx->R10); + Printf("r11 = %llx ", ctx->R11); + Printf("\n"); + Printf("r12 = %llx ", ctx->R12); + Printf("r13 = %llx ", ctx->R13); + Printf("r14 = %llx ", ctx->R14); + Printf("r15 = %llx ", ctx->R15); + Printf("\n"); +# elif defined(_M_IX86) + Report("Register values:\n"); + Printf("eax = %lx ", ctx->Eax); + Printf("ebx = %lx ", ctx->Ebx); + Printf("ecx = %lx ", ctx->Ecx); + Printf("edx = %lx ", ctx->Edx); + Printf("\n"); + Printf("edi = %lx ", ctx->Edi); + Printf("esi = %lx ", ctx->Esi); + Printf("ebp = %lx ", ctx->Ebp); + Printf("esp = %lx ", ctx->Esp); + Printf("\n"); +# elif defined(_M_ARM64) + Report("Register values:\n"); + for (int i = 0; i <= 30; i++) { + Printf("x%d%s = %llx", i < 10 ? " " : "", ctx->X[i]); + if (i % 4 == 3) + Printf("\n"); + } +# else + // TODO + (void)ctx; +# endif } int SignalContext::GetType() const { diff --git a/lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.cpp b/lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.cpp @@ -1,101 +0,0 @@ -//===-- sanitizer_win_dll_thunk.cpp ---------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// This file defines a family of thunks that should be statically linked into -// the DLLs that have instrumentation in order to delegate the calls to the -// shared runtime that lives in the main binary. -// See https://github.com/google/sanitizers/issues/209 for the details. -//===----------------------------------------------------------------------===// - -#ifdef SANITIZER_DLL_THUNK -#include "sanitizer_win_defs.h" -#include "sanitizer_win_dll_thunk.h" -#include "interception/interception.h" - -extern "C" { -void *WINAPI GetModuleHandleA(const char *module_name); -void abort(); -} - -namespace __sanitizer { -uptr dllThunkGetRealAddrOrDie(const char *name) { - uptr ret = - __interception::InternalGetProcAddress((void *)GetModuleHandleA(0), name); - if (!ret) - abort(); - return ret; -} - -int dllThunkIntercept(const char* main_function, uptr dll_function) { - uptr wrapper = dllThunkGetRealAddrOrDie(main_function); - if (!__interception::OverrideFunction(dll_function, wrapper, 0)) - abort(); - return 0; -} - -int dllThunkInterceptWhenPossible(const char* main_function, - const char* default_function, uptr dll_function) { - uptr wrapper = __interception::InternalGetProcAddress( - (void *)GetModuleHandleA(0), main_function); - if (!wrapper) - wrapper = dllThunkGetRealAddrOrDie(default_function); - if (!__interception::OverrideFunction(dll_function, wrapper, 0)) - abort(); - return 0; -} -} // namespace __sanitizer - -// Include Sanitizer Common interface. -#define INTERFACE_FUNCTION(Name) INTERCEPT_SANITIZER_FUNCTION(Name) -#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name) -#include "sanitizer_common_interface.inc" - -#pragma section(".DLLTH$A", read) -#pragma section(".DLLTH$Z", read) - -typedef void (*DllThunkCB)(); -extern "C" { -__declspec(allocate(".DLLTH$A")) DllThunkCB __start_dll_thunk; -__declspec(allocate(".DLLTH$Z")) DllThunkCB __stop_dll_thunk; -} - -// Disable compiler warnings that show up if we declare our own version -// of a compiler intrinsic (e.g. strlen). -#pragma warning(disable: 4391) -#pragma warning(disable: 4392) - -extern "C" int __dll_thunk_init() { - static bool flag = false; - // __dll_thunk_init is expected to be called by only one thread. - if (flag) return 0; - flag = true; - - for (DllThunkCB *it = &__start_dll_thunk; it < &__stop_dll_thunk; ++it) - if (*it) - (*it)(); - - // In DLLs, the callbacks are expected to return 0, - // otherwise CRT initialization fails. - return 0; -} - -// We want to call dll_thunk_init before C/C++ initializers / constructors are -// executed, otherwise functions like memset might be invoked. -#pragma section(".CRT$XIB", long, read) -__declspec(allocate(".CRT$XIB")) int (*__dll_thunk_preinit)() = - __dll_thunk_init; - -static void WINAPI dll_thunk_thread_init(void *mod, unsigned long reason, - void *reserved) { - if (reason == /*DLL_PROCESS_ATTACH=*/1) __dll_thunk_init(); -} - -#pragma section(".CRT$XLAB", long, read) -__declspec(allocate(".CRT$XLAB")) void (WINAPI *__dll_thunk_tls_init)(void *, - unsigned long, void *) = dll_thunk_thread_init; - -#endif // SANITIZER_DLL_THUNK diff --git a/lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.h b/lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.h @@ -1,181 +0,0 @@ -//===-- sanitizer_win_dll_thunk.h -----------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// This header provide helper macros to delegate calls to the shared runtime -// that lives in the main executable. It should be included to dll_thunks that -// will be linked to the dlls, when the sanitizer is a static library included -// in the main executable. -//===----------------------------------------------------------------------===// -#ifndef SANITIZER_WIN_DLL_THUNK_H -#define SANITIZER_WIN_DLL_THUNK_H -#include "sanitizer_internal_defs.h" - -namespace __sanitizer { -uptr dllThunkGetRealAddrOrDie(const char *name); - -int dllThunkIntercept(const char* main_function, uptr dll_function); - -int dllThunkInterceptWhenPossible(const char* main_function, - const char* default_function, uptr dll_function); -} - -extern "C" int __dll_thunk_init(); - -// ----------------- Function interception helper macros -------------------- // -// Override dll_function with main_function from main executable. -#define INTERCEPT_OR_DIE(main_function, dll_function) \ - static int intercept_##dll_function() { \ - return __sanitizer::dllThunkIntercept(main_function, (__sanitizer::uptr) \ - dll_function); \ - } \ - __pragma(section(".DLLTH$M", long, read)) \ - __declspec(allocate(".DLLTH$M")) int (*__dll_thunk_##dll_function)() = \ - intercept_##dll_function; - -// Try to override dll_function with main_function from main executable. -// If main_function is not present, override dll_function with default_function. -#define INTERCEPT_WHEN_POSSIBLE(main_function, default_function, dll_function) \ - static int intercept_##dll_function() { \ - return __sanitizer::dllThunkInterceptWhenPossible(main_function, \ - default_function, (__sanitizer::uptr)dll_function); \ - } \ - __pragma(section(".DLLTH$M", long, read)) \ - __declspec(allocate(".DLLTH$M")) int (*__dll_thunk_##dll_function)() = \ - intercept_##dll_function; - -// -------------------- Function interception macros ------------------------ // -// Special case of hooks -- ASan own interface functions. Those are only called -// after __asan_init, thus an empty implementation is sufficient. -#define INTERCEPT_SANITIZER_FUNCTION(name) \ - extern "C" __declspec(noinline) void name() { \ - volatile int prevent_icf = (__LINE__ << 8) ^ __COUNTER__; \ - static const char function_name[] = #name; \ - for (const char* ptr = &function_name[0]; *ptr; ++ptr) \ - prevent_icf ^= *ptr; \ - (void)prevent_icf; \ - __debugbreak(); \ - } \ - INTERCEPT_OR_DIE(#name, name) - -// Special case of hooks -- Weak functions, could be redefined in the main -// executable, but that is not necessary, so we shouldn't die if we can not find -// a reference. Instead, when the function is not present in the main executable -// we consider the default impl provided by asan library. -#define INTERCEPT_SANITIZER_WEAK_FUNCTION(name) \ - extern "C" __declspec(noinline) void name() { \ - volatile int prevent_icf = (__LINE__ << 8) ^ __COUNTER__; \ - static const char function_name[] = #name; \ - for (const char* ptr = &function_name[0]; *ptr; ++ptr) \ - prevent_icf ^= *ptr; \ - (void)prevent_icf; \ - __debugbreak(); \ - } \ - INTERCEPT_WHEN_POSSIBLE(#name, STRINGIFY(WEAK_EXPORT_NAME(name)), name) - -// We can't define our own version of strlen etc. because that would lead to -// link-time or even type mismatch errors. Instead, we can declare a function -// just to be able to get its address. Me may miss the first few calls to the -// functions since it can be called before __dll_thunk_init, but that would lead -// to false negatives in the startup code before user's global initializers, -// which isn't a big deal. -#define INTERCEPT_LIBRARY_FUNCTION(name) \ - extern "C" void name(); \ - INTERCEPT_OR_DIE(STRINGIFY(WRAP(name)), name) - -// Use these macros for functions that could be called before __dll_thunk_init() -// is executed and don't lead to errors if defined (free, malloc, etc). -#define INTERCEPT_WRAP_V_V(name) \ - extern "C" void name() { \ - typedef decltype(name) *fntype; \ - static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \ - fn(); \ - } \ - INTERCEPT_OR_DIE(#name, name); - -#define INTERCEPT_WRAP_V_W(name) \ - extern "C" void name(void *arg) { \ - typedef decltype(name) *fntype; \ - static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \ - fn(arg); \ - } \ - INTERCEPT_OR_DIE(#name, name); - -#define INTERCEPT_WRAP_V_WW(name) \ - extern "C" void name(void *arg1, void *arg2) { \ - typedef decltype(name) *fntype; \ - static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \ - fn(arg1, arg2); \ - } \ - INTERCEPT_OR_DIE(#name, name); - -#define INTERCEPT_WRAP_V_WWW(name) \ - extern "C" void name(void *arg1, void *arg2, void *arg3) { \ - typedef decltype(name) *fntype; \ - static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \ - fn(arg1, arg2, arg3); \ - } \ - INTERCEPT_OR_DIE(#name, name); - -#define INTERCEPT_WRAP_W_V(name) \ - extern "C" void *name() { \ - typedef decltype(name) *fntype; \ - static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \ - return fn(); \ - } \ - INTERCEPT_OR_DIE(#name, name); - -#define INTERCEPT_WRAP_W_W(name) \ - extern "C" void *name(void *arg) { \ - typedef decltype(name) *fntype; \ - static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \ - return fn(arg); \ - } \ - INTERCEPT_OR_DIE(#name, name); - -#define INTERCEPT_WRAP_W_WW(name) \ - extern "C" void *name(void *arg1, void *arg2) { \ - typedef decltype(name) *fntype; \ - static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \ - return fn(arg1, arg2); \ - } \ - INTERCEPT_OR_DIE(#name, name); - -#define INTERCEPT_WRAP_W_WWW(name) \ - extern "C" void *name(void *arg1, void *arg2, void *arg3) { \ - typedef decltype(name) *fntype; \ - static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \ - return fn(arg1, arg2, arg3); \ - } \ - INTERCEPT_OR_DIE(#name, name); - -#define INTERCEPT_WRAP_W_WWWW(name) \ - extern "C" void *name(void *arg1, void *arg2, void *arg3, void *arg4) { \ - typedef decltype(name) *fntype; \ - static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \ - return fn(arg1, arg2, arg3, arg4); \ - } \ - INTERCEPT_OR_DIE(#name, name); - -#define INTERCEPT_WRAP_W_WWWWW(name) \ - extern "C" void *name(void *arg1, void *arg2, void *arg3, void *arg4, \ - void *arg5) { \ - typedef decltype(name) *fntype; \ - static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \ - return fn(arg1, arg2, arg3, arg4, arg5); \ - } \ - INTERCEPT_OR_DIE(#name, name); - -#define INTERCEPT_WRAP_W_WWWWWW(name) \ - extern "C" void *name(void *arg1, void *arg2, void *arg3, void *arg4, \ - void *arg5, void *arg6) { \ - typedef decltype(name) *fntype; \ - static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \ - return fn(arg1, arg2, arg3, arg4, arg5, arg6); \ - } \ - INTERCEPT_OR_DIE(#name, name); - -#endif // SANITIZER_WIN_DLL_THUNK_H diff --git a/lib/tsan/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cpp b/lib/tsan/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cpp @@ -1,26 +0,0 @@ -//===-- santizer_win_dynamic_runtime_thunk.cpp ----------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file defines things that need to be present in the application modules -// to interact with Sanitizer Common, when it is included in a dll. -// -//===----------------------------------------------------------------------===// -#ifdef SANITIZER_DYNAMIC_RUNTIME_THUNK -#define SANITIZER_IMPORT_INTERFACE 1 -#include "sanitizer_win_defs.h" -// Define weak alias for all weak functions imported from sanitizer common. -#define INTERFACE_FUNCTION(Name) -#define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name) -#include "sanitizer_common_interface.inc" -#endif // SANITIZER_DYNAMIC_RUNTIME_THUNK - -namespace __sanitizer { -// Add one, otherwise unused, external symbol to this object file so that the -// Visual C++ linker includes it and reads the .drective section. -void ForceWholeArchiveIncludeForSanitizerCommon() {} -} diff --git a/lib/tsan/sanitizer_common/sanitizer_win_immortalize.h b/lib/tsan/sanitizer_common/sanitizer_win_immortalize.h @@ -0,0 +1,71 @@ +//===-- sanitizer_win_immortalize.h ---------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is shared between AddressSanitizer, and interception. +// +// Windows-specific thread-safe and pre-CRT global initialization safe +// infrastructure to create an object whose destructor is never called. +//===----------------------------------------------------------------------===// +#if SANITIZER_WINDOWS +# pragma once +// Requires including sanitizer_placement_new.h (which is not allowed to be +// included in headers). + +# include "sanitizer_win_defs.h" +// These types are required to satisfy XFG which requires that the names of the +// types for indirect calls to be correct as well as the name of the original +// type for any typedefs. + +// TODO: There must be a better way to do this +# ifndef _WINDOWS_ +typedef void* PVOID; +typedef int BOOL; +typedef union _RTL_RUN_ONCE { + PVOID ptr; +} INIT_ONCE, *PINIT_ONCE; + +extern "C" { +__declspec(dllimport) int WINAPI InitOnceExecuteOnce( + PINIT_ONCE, BOOL(WINAPI*)(PINIT_ONCE, PVOID, PVOID*), void*, void*); +} +# endif + +namespace __sanitizer { +template <class Ty> +BOOL WINAPI immortalize_impl(PINIT_ONCE, PVOID storage_ptr, PVOID*) noexcept { + // Ty must provide a placement new operator + new (storage_ptr) Ty(); + return 1; +} + +template <class Ty, typename Arg> +BOOL WINAPI immortalize_impl(PINIT_ONCE, PVOID storage_ptr, + PVOID* param) noexcept { + // Ty must provide a placement new operator + new (storage_ptr) Ty(*((Arg*)param)); + return 1; +} + +template <class Ty> +Ty& immortalize() { // return a reference to an object that will live forever + static INIT_ONCE flag; + alignas(Ty) static unsigned char storage[sizeof(Ty)]; + InitOnceExecuteOnce(&flag, immortalize_impl<Ty>, &storage, nullptr); + return reinterpret_cast<Ty&>(storage); +} + +template <class Ty, typename Arg> +Ty& immortalize( + Arg arg) { // return a reference to an object that will live forever + static INIT_ONCE flag; + alignas(Ty) static unsigned char storage[sizeof(Ty)]; + InitOnceExecuteOnce(&flag, immortalize_impl<Ty, Arg>, &storage, &arg); + return reinterpret_cast<Ty&>(storage); +} +} // namespace __sanitizer +#endif // SANITIZER_WINDOWS diff --git a/lib/tsan/sanitizer_common/sanitizer_win_interception.cpp b/lib/tsan/sanitizer_common/sanitizer_win_interception.cpp @@ -0,0 +1,156 @@ +//===-- sanitizer_win_interception.cpp -------------------- --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Windows-specific export surface to provide interception for parts of the +// runtime that are always statically linked, both for overriding user-defined +// functions as well as registering weak functions that the ASAN runtime should +// use over defaults. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" +#if SANITIZER_WINDOWS +# include <stddef.h> + +# include "interception/interception.h" +# include "sanitizer_addrhashmap.h" +# include "sanitizer_common.h" +# include "sanitizer_internal_defs.h" +# include "sanitizer_placement_new.h" +# include "sanitizer_win_immortalize.h" +# include "sanitizer_win_interception.h" + +using namespace __sanitizer; + +extern "C" void *__ImageBase; + +namespace __sanitizer { + +static uptr GetSanitizerDllExport(const char *export_name) { + const uptr function_address = + __interception::InternalGetProcAddress(&__ImageBase, export_name); + if (function_address == 0) { + Report("ERROR: Failed to find sanitizer DLL export '%s'\n", export_name); + CHECK("Failed to find sanitizer DLL export" && 0); + } + return function_address; +} + +struct WeakCallbackList { + explicit constexpr WeakCallbackList(RegisterWeakFunctionCallback cb) + : callback(cb), next(nullptr) {} + + static void *operator new(size_t size) { return InternalAlloc(size); } + + static void operator delete(void *p) { InternalFree(p); } + + RegisterWeakFunctionCallback callback; + WeakCallbackList *next; +}; +using WeakCallbackMap = AddrHashMap<WeakCallbackList *, 11>; + +static WeakCallbackMap *GetWeakCallbackMap() { + return &immortalize<WeakCallbackMap>(); +} + +void AddRegisterWeakFunctionCallback(uptr export_address, + RegisterWeakFunctionCallback cb) { + WeakCallbackMap::Handle h_find_or_create(GetWeakCallbackMap(), export_address, + false, true); + CHECK(h_find_or_create.exists()); + if (h_find_or_create.created()) { + *h_find_or_create = new WeakCallbackList(cb); + } else { + (*h_find_or_create)->next = new WeakCallbackList(cb); + } +} + +static void RunWeakFunctionCallbacks(uptr export_address) { + WeakCallbackMap::Handle h_find(GetWeakCallbackMap(), export_address, false, + false); + if (!h_find.exists()) { + return; + } + + WeakCallbackList *list = *h_find; + do { + list->callback(); + } while ((list = list->next)); +} + +} // namespace __sanitizer + +extern "C" __declspec(dllexport) bool __cdecl __sanitizer_override_function( + const char *export_name, const uptr user_function, + uptr *const old_user_function) { + CHECK(export_name); + CHECK(user_function); + + const uptr sanitizer_function = GetSanitizerDllExport(export_name); + + const bool function_overridden = __interception::OverrideFunction( + user_function, sanitizer_function, old_user_function); + if (!function_overridden) { + Report( + "ERROR: Failed to override local function at '%p' with sanitizer " + "function '%s'\n", + user_function, export_name); + CHECK("Failed to replace local function with sanitizer version." && 0); + } + + return function_overridden; +} + +extern "C" + __declspec(dllexport) bool __cdecl __sanitizer_override_function_by_addr( + const uptr source_function, const uptr target_function, + uptr *const old_target_function) { + CHECK(source_function); + CHECK(target_function); + + const bool function_overridden = __interception::OverrideFunction( + target_function, source_function, old_target_function); + if (!function_overridden) { + Report( + "ERROR: Failed to override function at '%p' with function at " + "'%p'\n", + target_function, source_function); + CHECK("Failed to apply function override." && 0); + } + + return function_overridden; +} + +extern "C" + __declspec(dllexport) bool __cdecl __sanitizer_register_weak_function( + const char *export_name, const uptr user_function, + uptr *const old_user_function) { + CHECK(export_name); + CHECK(user_function); + + const uptr sanitizer_function = GetSanitizerDllExport(export_name); + + const bool function_overridden = __interception::OverrideFunction( + sanitizer_function, user_function, old_user_function); + if (!function_overridden) { + Report( + "ERROR: Failed to register local function at '%p' to be used in " + "place of sanitizer function '%s'\n.", + user_function, export_name); + CHECK("Failed to register weak function." && 0); + } + + // Note that thread-safety of RunWeakFunctionCallbacks in InitializeFlags + // depends on __sanitizer_register_weak_functions being called during the + // loader lock. + RunWeakFunctionCallbacks(sanitizer_function); + + return function_overridden; +} + +#endif // SANITIZER_WINDOWS diff --git a/lib/tsan/sanitizer_common/sanitizer_win_interception.h b/lib/tsan/sanitizer_common/sanitizer_win_interception.h @@ -0,0 +1,32 @@ +//===-- sanitizer_win_interception.h ---------------------- --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Windows-specific export surface to provide interception for parts of the +// runtime that are always statically linked, both for overriding user-defined +// functions as well as registering weak functions that the ASAN runtime should +// use over defaults. +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_WIN_INTERCEPTION_H +#define SANITIZER_WIN_INTERCEPTION_H + +#include "sanitizer_platform.h" +#if SANITIZER_WINDOWS + +# include "sanitizer_common.h" +# include "sanitizer_internal_defs.h" + +namespace __sanitizer { +using RegisterWeakFunctionCallback = void (*)(); +void AddRegisterWeakFunctionCallback(uptr export_address, + RegisterWeakFunctionCallback cb); +} // namespace __sanitizer + +#endif // SANITIZER_WINDOWS +#endif // SANITIZER_WIN_INTERCEPTION_H +\ No newline at end of file diff --git a/lib/tsan/sanitizer_common/sanitizer_win_weak_interception.cpp b/lib/tsan/sanitizer_common/sanitizer_win_weak_interception.cpp @@ -1,94 +0,0 @@ -//===-- sanitizer_win_weak_interception.cpp -------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// This module should be included in the sanitizer when it is implemented as a -// shared library on Windows (dll), in order to delegate the calls of weak -// functions to the implementation in the main executable when a strong -// definition is provided. -//===----------------------------------------------------------------------===// - -#include "sanitizer_platform.h" -#if SANITIZER_WINDOWS && SANITIZER_DYNAMIC -#include "sanitizer_win_weak_interception.h" -#include "sanitizer_allocator_interface.h" -#include "sanitizer_interface_internal.h" -#include "sanitizer_win_defs.h" -#include "interception/interception.h" - -extern "C" { -void *WINAPI GetModuleHandleA(const char *module_name); -void abort(); -} - -namespace __sanitizer { -// Try to get a pointer to real_function in the main module and override -// dll_function with that pointer. If the function isn't found, nothing changes. -int interceptWhenPossible(uptr dll_function, const char *real_function) { - uptr real = __interception::InternalGetProcAddress( - (void *)GetModuleHandleA(0), real_function); - if (real && !__interception::OverrideFunction((uptr)dll_function, real, 0)) - abort(); - return 0; -} -} // namespace __sanitizer - -// Declare weak hooks. -extern "C" { -void __sanitizer_on_print(const char *str); -void __sanitizer_weak_hook_memcmp(uptr called_pc, const void *s1, - const void *s2, uptr n, int result); -void __sanitizer_weak_hook_strcmp(uptr called_pc, const char *s1, - const char *s2, int result); -void __sanitizer_weak_hook_strncmp(uptr called_pc, const char *s1, - const char *s2, uptr n, int result); -void __sanitizer_weak_hook_strstr(uptr called_pc, const char *s1, - const char *s2, char *result); -} - -// Include Sanitizer Common interface. -#define INTERFACE_FUNCTION(Name) -#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name) -#include "sanitizer_common_interface.inc" - -#pragma section(".WEAK$A", read) -#pragma section(".WEAK$Z", read) - -typedef void (*InterceptCB)(); -extern "C" { -__declspec(allocate(".WEAK$A")) InterceptCB __start_weak_list; -__declspec(allocate(".WEAK$Z")) InterceptCB __stop_weak_list; -} - -static int weak_intercept_init() { - static bool flag = false; - // weak_interception_init is expected to be called by only one thread. - if (flag) return 0; - flag = true; - - for (InterceptCB *it = &__start_weak_list; it < &__stop_weak_list; ++it) - if (*it) - (*it)(); - - // In DLLs, the callbacks are expected to return 0, - // otherwise CRT initialization fails. - return 0; -} - -#pragma section(".CRT$XIB", long, read) -__declspec(allocate(".CRT$XIB")) int (*__weak_intercept_preinit)() = - weak_intercept_init; - -static void WINAPI weak_intercept_thread_init(void *mod, unsigned long reason, - void *reserved) { - if (reason == /*DLL_PROCESS_ATTACH=*/1) weak_intercept_init(); -} - -#pragma section(".CRT$XLAB", long, read) -__declspec(allocate(".CRT$XLAB")) void(WINAPI *__weak_intercept_tls_init)( - void *, unsigned long, void *) = weak_intercept_thread_init; - -#endif // SANITIZER_WINDOWS && SANITIZER_DYNAMIC diff --git a/lib/tsan/sanitizer_common/sanitizer_win_weak_interception.h b/lib/tsan/sanitizer_common/sanitizer_win_weak_interception.h @@ -1,32 +0,0 @@ -//===-- sanitizer_win_weak_interception.h ---------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// This header provide helper macros to delegate calls of weak functions to the -// implementation in the main executable when a strong definition is present. -//===----------------------------------------------------------------------===// -#ifndef SANITIZER_WIN_WEAK_INTERCEPTION_H -#define SANITIZER_WIN_WEAK_INTERCEPTION_H -#include "sanitizer_internal_defs.h" - -namespace __sanitizer { -int interceptWhenPossible(uptr dll_function, const char *real_function); -} - -// ----------------- Function interception helper macros -------------------- // -// Weak functions, could be redefined in the main executable, but that is not -// necessary, so we shouldn't die if we can not find a reference. -#define INTERCEPT_WEAK(Name) interceptWhenPossible((uptr) Name, #Name); - -#define INTERCEPT_SANITIZER_WEAK_FUNCTION(Name) \ - static int intercept_##Name() { \ - return __sanitizer::interceptWhenPossible((__sanitizer::uptr) Name, #Name);\ - } \ - __pragma(section(".WEAK$M", long, read)) \ - __declspec(allocate(".WEAK$M")) int (*__weak_intercept_##Name)() = \ - intercept_##Name; - -#endif // SANITIZER_WIN_WEAK_INTERCEPTION_H diff --git a/lib/tsan/tsan_interceptors_mac.cpp b/lib/tsan/tsan_interceptors_mac.cpp @@ -14,22 +14,21 @@ #include "sanitizer_common/sanitizer_platform.h" #if SANITIZER_APPLE -#include "interception/interception.h" -#include "tsan_interceptors.h" -#include "tsan_interface.h" -#include "tsan_interface_ann.h" -#include "tsan_spinlock_defs_mac.h" -#include "sanitizer_common/sanitizer_addrhashmap.h" - -#include <errno.h> -#include <libkern/OSAtomic.h> -#include <objc/objc-sync.h> -#include <os/lock.h> -#include <sys/ucontext.h> - -#if defined(__has_include) && __has_include(<xpc/xpc.h>) -#include <xpc/xpc.h> -#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) +# include <errno.h> +# include <libkern/OSAtomic.h> +# include <objc/objc-sync.h> +# include <os/lock.h> +# include <sys/ucontext.h> + +# include "interception/interception.h" +# include "sanitizer_common/sanitizer_addrhashmap.h" +# include "tsan_interceptors.h" +# include "tsan_interface.h" +# include "tsan_interface_ann.h" + +# if defined(__has_include) && __has_include(<xpc/xpc.h>) +# include <xpc/xpc.h> +# endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) typedef long long_t; @@ -49,51 +48,55 @@ static constexpr morder kMacOrderBarrier = mo_acq_rel; static constexpr morder kMacOrderNonBarrier = mo_acq_rel; static constexpr morder kMacFailureOrder = mo_relaxed; -#define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ - TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ - SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ - return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \ - } - -#define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ - TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ - SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ - return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \ - } +# define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ + TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ + return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \ + } -#define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ - TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ - SCOPED_TSAN_INTERCEPTOR(f, ptr); \ - return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \ - } +# define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, \ + mo) \ + TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ + return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \ + } -#define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \ - mo) \ - TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ - SCOPED_TSAN_INTERCEPTOR(f, ptr); \ - return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \ - } +# define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \ + mo) \ + TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, ptr); \ + return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \ + } -#define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \ - m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ - kMacOrderNonBarrier) \ - m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ - kMacOrderBarrier) \ - m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \ - kMacOrderNonBarrier) \ - m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \ - kMacOrderBarrier) - -#define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \ - m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ - kMacOrderNonBarrier) \ - m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ - kMacOrderBarrier) \ - m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \ - kMacOrderNonBarrier) \ - m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \ - __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) +# define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \ + mo) \ + TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, ptr); \ + return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \ + } +# define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \ + m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ + kMacOrderNonBarrier) \ + m(int32_t, int32_t, a32, f##32##Barrier, \ + __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) \ + m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \ + kMacOrderNonBarrier) \ + m(int64_t, int64_t, a64, f##64##Barrier, \ + __tsan_atomic64_##tsan_atomic_f, kMacOrderBarrier) + +# define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \ + m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ + kMacOrderNonBarrier) \ + m(int32_t, uint32_t, a32, f##32##Barrier, \ + __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) \ + m_orig(int32_t, uint32_t, a32, f##32##Orig, \ + __tsan_atomic32_##tsan_atomic_f, kMacOrderNonBarrier) \ + m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \ + __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) + +# pragma clang diagnostic push // OSAtomic* deprecation +# pragma clang diagnostic ignored "-Wdeprecated-declarations" OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add, OSATOMIC_INTERCEPTOR_PLUS_X) OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add, @@ -106,23 +109,26 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and, OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor, OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) +# pragma clang diagnostic pop // OSAtomic* deprecation + +# define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \ + TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \ + return tsan_atomic_f##_compare_exchange_strong( \ + (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ + kMacOrderNonBarrier, kMacFailureOrder); \ + } \ + \ + TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \ + t volatile *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \ + return tsan_atomic_f##_compare_exchange_strong( \ + (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ + kMacOrderBarrier, kMacFailureOrder); \ + } -#define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \ - TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \ - SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \ - return tsan_atomic_f##_compare_exchange_strong( \ - (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ - kMacOrderNonBarrier, kMacFailureOrder); \ - } \ - \ - TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \ - t volatile *ptr) { \ - SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \ - return tsan_atomic_f##_compare_exchange_strong( \ - (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ - kMacOrderBarrier, kMacFailureOrder); \ - } - +# pragma clang diagnostic push // OSAtomicCompareAndSwap* deprecation +# pragma clang diagnostic ignored "-Wdeprecated-declarations" OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int) OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64, long_t) @@ -132,24 +138,28 @@ OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32, int32_t) OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64, int64_t) +# pragma clang diagnostic pop // OSAtomicCompareAndSwap* deprecation + +# define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \ + TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \ + SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \ + volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \ + char bit = 0x80u >> (n & 7); \ + char mask = clear ? ~bit : bit; \ + char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \ + return orig_byte & bit; \ + } -#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \ - TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \ - SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \ - volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \ - char bit = 0x80u >> (n & 7); \ - char mask = clear ? ~bit : bit; \ - char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \ - return orig_byte & bit; \ - } - -#define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \ - OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \ - OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier) +# define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \ + OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \ + OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier) +# pragma clang diagnostic push // OSAtomicTestAnd* deprecation +# pragma clang diagnostic ignored "-Wdeprecated-declarations" OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false) OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and, true) +# pragma clang diagnostic pop // OSAtomicTestAnd* deprecation TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item, size_t offset) { @@ -161,12 +171,13 @@ TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item, TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) { SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset); void *item = REAL(OSAtomicDequeue)(list, offset); - if (item) __tsan_acquire(item); + if (item) + __tsan_acquire(item); return item; } // OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X. -#if !SANITIZER_IOS +# if !SANITIZER_IOS TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item, size_t offset) { @@ -179,11 +190,22 @@ TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list, size_t offset) { SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset); void *item = REAL(OSAtomicFifoDequeue)(list, offset); - if (item) __tsan_acquire(item); + if (item) + __tsan_acquire(item); return item; } -#endif +# endif + +// If `OSSPINLOCK_USE_INLINED=1` is set, then SDK headers don't declare these +// as functions, but macros that call non-deprecated APIs. Undefine these +// macros so they don't interfere with the interceptor machinery. +# undef OSSpinLockLock +# undef OSSpinLockTry +# undef OSSpinLockUnlock + +# pragma clang diagnostic push // OSSpinLock* deprecation +# pragma clang diagnostic ignored "-Wdeprecated-declarations" TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) { CHECK(!cur_thread()->is_dead); @@ -216,6 +238,7 @@ TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) { Release(thr, pc, (uptr)lock); REAL(OSSpinLockUnlock)(lock); } +# pragma clang diagnostic pop // OSSpinLock* deprecation TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) { CHECK(!cur_thread()->is_dead); @@ -288,7 +311,7 @@ TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) { REAL(os_unfair_lock_unlock)(lock); } -#if defined(__has_include) && __has_include(<xpc/xpc.h>) +# if defined(__has_include) && __has_include(<xpc/xpc.h>) TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler, xpc_connection_t connection, xpc_handler_t handler) { @@ -342,7 +365,7 @@ TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) { REAL(xpc_connection_cancel)(connection); } -#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) +# endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) // Determines whether the Obj-C object pointer is a tagged pointer. Tagged // pointers encode the object data directly in their pointer bits and do not @@ -365,7 +388,7 @@ static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) { Map::Handle h(&Addresses, addr); if (h.created()) { ThreadIgnoreBegin(thr, pc); - *h = (uptr) user_alloc(thr, pc, /*size=*/1); + *h = (uptr)user_alloc(thr, pc, /*size=*/1); ThreadIgnoreEnd(thr); } return *h; @@ -383,7 +406,8 @@ static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) { TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) { SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj); - if (!obj) return REAL(objc_sync_enter)(obj); + if (!obj) + return REAL(objc_sync_enter)(obj); uptr addr = SyncAddressForObjCObject(obj, thr, pc); MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant); int result = REAL(objc_sync_enter)(obj); @@ -394,11 +418,13 @@ TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) { TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) { SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj); - if (!obj) return REAL(objc_sync_exit)(obj); + if (!obj) + return REAL(objc_sync_exit)(obj); uptr addr = SyncAddressForObjCObject(obj, thr, pc); MutexUnlock(thr, pc, addr); int result = REAL(objc_sync_exit)(obj); - if (result != OBJC_SYNC_SUCCESS) MutexInvalidAccess(thr, pc, addr); + if (result != OBJC_SYNC_SUCCESS) + MutexInvalidAccess(thr, pc, addr); return result; } @@ -429,7 +455,7 @@ TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) { // On macOS, libc++ is always linked dynamically, so intercepting works the // usual way. -#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR +# define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR namespace { struct fake_shared_weak_count { diff --git a/lib/tsan/tsan_interceptors_posix.cpp b/lib/tsan/tsan_interceptors_posix.cpp @@ -12,14 +12,15 @@ // sanitizer_common/sanitizer_common_interceptors.inc //===----------------------------------------------------------------------===// +#include "sanitizer_common/sanitizer_allocator_dlsym.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_glibc_version.h" +#include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_platform_limits_netbsd.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" -#include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_posix.h" #include "sanitizer_common/sanitizer_stacktrace.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" @@ -96,7 +97,7 @@ extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v)); extern "C" int pthread_setspecific(unsigned key, const void *v); DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *) DECLARE_REAL(int, fflush, __sanitizer_FILE *fp) -DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size) +DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, usize size) DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) extern "C" int pthread_equal(void *t1, void *t2); extern "C" void *pthread_self(); @@ -252,6 +253,13 @@ SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionBegin() {} SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {} #endif +// FIXME: Use for `in_symbolizer()` as well. As-is we can't use +// `DlSymAllocator`, because it uses the primary allocator only. Symbolizer +// requires support of the secondary allocator for larger blocks. +struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> { + static bool UseImpl() { return (ctx && !ctx->initialized); } +}; + } // namespace __tsan static ThreadSignalContext *SigCtx(ThreadState *thr) { @@ -661,6 +669,8 @@ TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) { TSAN_INTERCEPTOR(void*, malloc, uptr size) { if (in_symbolizer()) return InternalAlloc(size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Allocate(size); void *p = 0; { SCOPED_INTERCEPTOR_RAW(malloc, size); @@ -678,12 +688,14 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { return user_memalign(thr, pc, align, sz); } -TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { +TSAN_INTERCEPTOR(void *, calloc, uptr n, uptr size) { if (in_symbolizer()) - return InternalCalloc(size, n); + return InternalCalloc(n, size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(n, size); void *p = 0; { - SCOPED_INTERCEPTOR_RAW(calloc, size, n); + SCOPED_INTERCEPTOR_RAW(calloc, n, size); p = user_calloc(thr, pc, size, n); } invoke_malloc_hook(p, n * size); @@ -693,6 +705,8 @@ TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { if (in_symbolizer()) return InternalRealloc(p, size); + if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(p)) + return DlsymAlloc::Realloc(p, size); if (p) invoke_free_hook(p); { @@ -703,13 +717,13 @@ TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { return p; } -TSAN_INTERCEPTOR(void*, reallocarray, void *p, uptr size, uptr n) { +TSAN_INTERCEPTOR(void *, reallocarray, void *p, uptr n, uptr size) { if (in_symbolizer()) - return InternalReallocArray(p, size, n); + return InternalReallocArray(p, n, size); if (p) invoke_free_hook(p); { - SCOPED_INTERCEPTOR_RAW(reallocarray, p, size, n); + SCOPED_INTERCEPTOR_RAW(reallocarray, p, n, size); p = user_reallocarray(thr, pc, p, size, n); } invoke_malloc_hook(p, size); @@ -717,20 +731,24 @@ TSAN_INTERCEPTOR(void*, reallocarray, void *p, uptr size, uptr n) { } TSAN_INTERCEPTOR(void, free, void *p) { - if (p == 0) + if (UNLIKELY(!p)) return; if (in_symbolizer()) return InternalFree(p); + if (DlsymAlloc::PointerIsMine(p)) + return DlsymAlloc::Free(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(free, p); user_free(thr, pc, p); } TSAN_INTERCEPTOR(void, cfree, void *p) { - if (p == 0) + if (UNLIKELY(!p)) return; if (in_symbolizer()) return InternalFree(p); + if (DlsymAlloc::PointerIsMine(p)) + return DlsymAlloc::Free(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(cfree, p); user_free(thr, pc, p); @@ -750,7 +768,7 @@ TSAN_INTERCEPTOR(char *, strcpy, char *dst, const char *src) { return REAL(strcpy)(dst, src); } -TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) { +TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, usize n) { SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n); uptr srclen = internal_strnlen(src, n); MemoryAccessRange(thr, pc, (uptr)dst, n, true); @@ -1097,7 +1115,7 @@ int internal_pthread_create(void *th, void *attr, void *(*callback)(void *), } int internal_pthread_join(void *th, void **ret) { ScopedIgnoreInterceptors ignore; - return REAL(pthread_join(th, ret)); + return REAL(pthread_join)(th, ret); } } // namespace __sanitizer @@ -1662,13 +1680,23 @@ TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) { #endif TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) { - va_list ap; - va_start(ap, oflag); - mode_t mode = va_arg(ap, int); - va_end(ap); + mode_t mode = 0; + if (OpenReadsVaArgs(oflag)) { + va_list ap; + va_start(ap, oflag); + mode = va_arg(ap, int); + va_end(ap); + } + SCOPED_TSAN_INTERCEPTOR(open, name, oflag, mode); READ_STRING(thr, pc, name, 0); - int fd = REAL(open)(name, oflag, mode); + + int fd; + if (OpenReadsVaArgs(oflag)) + fd = REAL(open)(name, oflag, mode); + else + fd = REAL(open)(name, oflag); + if (fd >= 0) FdFileCreate(thr, pc, fd); return fd; diff --git a/lib/tsan/tsan_interface.h b/lib/tsan/tsan_interface.h @@ -16,8 +16,8 @@ #define TSAN_INTERFACE_H #include <sanitizer_common/sanitizer_internal_defs.h> -using __sanitizer::uptr; using __sanitizer::tid_t; +using __sanitizer::uptr; // This header should NOT include any other headers. // All functions in this header are extern "C" and start with __tsan_. @@ -203,17 +203,18 @@ int __tsan_get_alloc_stack(uptr addr, uptr *trace, uptr size, int *thread_id, namespace __tsan { // These should match declarations from public tsan_interface_atomic.h header. -typedef unsigned char a8; +typedef unsigned char a8; typedef unsigned short a16; -typedef unsigned int a32; +typedef unsigned int a32; typedef unsigned long long a64; -#if !SANITIZER_GO && (defined(__SIZEOF_INT128__) \ - || (__clang_major__ * 100 + __clang_minor__ >= 302)) && \ +#if !SANITIZER_GO && \ + (defined(__SIZEOF_INT128__) || \ + (__clang_major__ * 100 + __clang_minor__ >= 302)) && \ !defined(__mips64) && !defined(__s390x__) __extension__ typedef __int128 a128; -# define __TSAN_HAS_INT128 1 +# define __TSAN_HAS_INT128 1 #else -# define __TSAN_HAS_INT128 0 +# define __TSAN_HAS_INT128 0 #endif // Part of ABI, do not change. @@ -231,180 +232,180 @@ struct ThreadState; extern "C" { SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_load(const volatile a8 *a, morder mo); +a8 __tsan_atomic8_load(const volatile a8 *a, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_load(const volatile a16 *a, morder mo); +a16 __tsan_atomic16_load(const volatile a16 *a, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_load(const volatile a32 *a, morder mo); +a32 __tsan_atomic32_load(const volatile a32 *a, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_load(const volatile a64 *a, morder mo); +a64 __tsan_atomic64_load(const volatile a64 *a, int mo); #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_load(const volatile a128 *a, morder mo); +a128 __tsan_atomic128_load(const volatile a128 *a, int mo); #endif SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo); +void __tsan_atomic8_store(volatile a8 *a, a8 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo); +void __tsan_atomic16_store(volatile a16 *a, a16 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo); +void __tsan_atomic32_store(volatile a32 *a, a32 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo); +void __tsan_atomic64_store(volatile a64 *a, a64 v, int mo); #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo); +void __tsan_atomic128_store(volatile a128 *a, a128 v, int mo); #endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo); +a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo); +a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo); +a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo); +a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, int mo); #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo); +a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, int mo); #endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo); +a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo); +a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo); +a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo); +a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, int mo); #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo); +a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, int mo); #endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo); +a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo); +a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo); +a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo); +a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, int mo); #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo); +a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, int mo); #endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo); +a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo); +a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo); +a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo); +a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, int mo); #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo); +a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, int mo); #endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo); +a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo); +a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo); +a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo); +a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, int mo); #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo); +a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, int mo); #endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo); +a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo); +a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo); +a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo); +a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, int mo); #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo); +a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, int mo); #endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo); +a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo); +a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo); +a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, int mo); SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo); +a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, int mo); #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo); +a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, int mo); #endif SANITIZER_INTERFACE_ATTRIBUTE -int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, - morder mo, morder fmo); +int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, int mo, + int fmo); SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v, - morder mo, morder fmo); + int mo, int fmo); SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v, - morder mo, morder fmo); + int mo, int fmo); SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v, - morder mo, morder fmo); + int mo, int fmo); #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v, - morder mo, morder fmo); + int mo, int fmo); #endif SANITIZER_INTERFACE_ATTRIBUTE -int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, morder mo, - morder fmo); +int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, int mo, + int fmo); SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v, - morder mo, morder fmo); + int mo, int fmo); SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v, - morder mo, morder fmo); + int mo, int fmo); SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v, - morder mo, morder fmo); + int mo, int fmo); #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v, - morder mo, morder fmo); + int mo, int fmo); #endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, morder mo, - morder fmo); +a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, int mo, + int fmo); SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, - morder mo, morder fmo); +a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, int mo, + int fmo); SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, - morder mo, morder fmo); +a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, int mo, + int fmo); SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, - morder mo, morder fmo); +a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, int mo, + int fmo); #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v, - morder mo, morder fmo); + int mo, int fmo); #endif SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_atomic_thread_fence(morder mo); +void __tsan_atomic_thread_fence(int mo); SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_atomic_signal_fence(morder mo); +void __tsan_atomic_signal_fence(int mo); SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a); diff --git a/lib/tsan/tsan_interface_atomic.cpp b/lib/tsan/tsan_interface_atomic.cpp @@ -18,9 +18,9 @@ // The following page contains more background information: // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ +#include "sanitizer_common/sanitizer_mutex.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_stacktrace.h" -#include "sanitizer_common/sanitizer_mutex.h" #include "tsan_flags.h" #include "tsan_interface.h" #include "tsan_rtl.h" @@ -34,8 +34,8 @@ static StaticSpinMutex mutex128; #if SANITIZER_DEBUG static bool IsLoadOrder(morder mo) { - return mo == mo_relaxed || mo == mo_consume - || mo == mo_acquire || mo == mo_seq_cst; + return mo == mo_relaxed || mo == mo_consume || mo == mo_acquire || + mo == mo_seq_cst; } static bool IsStoreOrder(morder mo) { @@ -48,42 +48,49 @@ static bool IsReleaseOrder(morder mo) { } static bool IsAcquireOrder(morder mo) { - return mo == mo_consume || mo == mo_acquire - || mo == mo_acq_rel || mo == mo_seq_cst; + return mo == mo_consume || mo == mo_acquire || mo == mo_acq_rel || + mo == mo_seq_cst; } static bool IsAcqRelOrder(morder mo) { return mo == mo_acq_rel || mo == mo_seq_cst; } -template<typename T> T func_xchg(volatile T *v, T op) { +template <typename T> +T func_xchg(volatile T *v, T op) { T res = __sync_lock_test_and_set(v, op); // __sync_lock_test_and_set does not contain full barrier. __sync_synchronize(); return res; } -template<typename T> T func_add(volatile T *v, T op) { +template <typename T> +T func_add(volatile T *v, T op) { return __sync_fetch_and_add(v, op); } -template<typename T> T func_sub(volatile T *v, T op) { +template <typename T> +T func_sub(volatile T *v, T op) { return __sync_fetch_and_sub(v, op); } -template<typename T> T func_and(volatile T *v, T op) { +template <typename T> +T func_and(volatile T *v, T op) { return __sync_fetch_and_and(v, op); } -template<typename T> T func_or(volatile T *v, T op) { +template <typename T> +T func_or(volatile T *v, T op) { return __sync_fetch_and_or(v, op); } -template<typename T> T func_xor(volatile T *v, T op) { +template <typename T> +T func_xor(volatile T *v, T op) { return __sync_fetch_and_xor(v, op); } -template<typename T> T func_nand(volatile T *v, T op) { +template <typename T> +T func_nand(volatile T *v, T op) { // clang does not support __sync_fetch_and_nand. T cmp = *v; for (;;) { @@ -95,7 +102,8 @@ template<typename T> T func_nand(volatile T *v, T op) { } } -template<typename T> T func_cas(volatile T *v, T cmp, T xch) { +template <typename T> +T func_cas(volatile T *v, T cmp, T xch) { return __sync_val_compare_and_swap(v, cmp, xch); } @@ -103,8 +111,8 @@ template<typename T> T func_cas(volatile T *v, T cmp, T xch) { // Atomic ops are executed under tsan internal mutex, // here we assume that the atomic variables are not accessed // from non-instrumented code. -#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO \ - && __TSAN_HAS_INT128 +#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO && \ + __TSAN_HAS_INT128 a128 func_xchg(volatile a128 *v, a128 op) { SpinMutexLock lock(&mutex128); a128 cmp = *v; @@ -197,89 +205,24 @@ static atomic_uint64_t *to_atomic(const volatile a64 *a) { static memory_order to_mo(morder mo) { switch (mo) { - case mo_relaxed: return memory_order_relaxed; - case mo_consume: return memory_order_consume; - case mo_acquire: return memory_order_acquire; - case mo_release: return memory_order_release; - case mo_acq_rel: return memory_order_acq_rel; - case mo_seq_cst: return memory_order_seq_cst; + case mo_relaxed: + return memory_order_relaxed; + case mo_consume: + return memory_order_consume; + case mo_acquire: + return memory_order_acquire; + case mo_release: + return memory_order_release; + case mo_acq_rel: + return memory_order_acq_rel; + case mo_seq_cst: + return memory_order_seq_cst; } DCHECK(0); return memory_order_seq_cst; } -template<typename T> -static T NoTsanAtomicLoad(const volatile T *a, morder mo) { - return atomic_load(to_atomic(a), to_mo(mo)); -} - -#if __TSAN_HAS_INT128 && !SANITIZER_GO -static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) { - SpinMutexLock lock(&mutex128); - return *a; -} -#endif - -template <typename T> -static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) { - DCHECK(IsLoadOrder(mo)); - // This fast-path is critical for performance. - // Assume the access is atomic. - if (!IsAcquireOrder(mo)) { - MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), - kAccessRead | kAccessAtomic); - return NoTsanAtomicLoad(a, mo); - } - // Don't create sync object if it does not exist yet. For example, an atomic - // pointer is initialized to nullptr and then periodically acquire-loaded. - T v = NoTsanAtomicLoad(a, mo); - SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a); - if (s) { - SlotLocker locker(thr); - ReadLock lock(&s->mtx); - thr->clock.Acquire(s->clock); - // Re-read under sync mutex because we need a consistent snapshot - // of the value and the clock we acquire. - v = NoTsanAtomicLoad(a, mo); - } - MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessRead | kAccessAtomic); - return v; -} - -template<typename T> -static void NoTsanAtomicStore(volatile T *a, T v, morder mo) { - atomic_store(to_atomic(a), v, to_mo(mo)); -} - -#if __TSAN_HAS_INT128 && !SANITIZER_GO -static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) { - SpinMutexLock lock(&mutex128); - *a = v; -} -#endif - -template <typename T> -static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - DCHECK(IsStoreOrder(mo)); - MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic); - // This fast-path is critical for performance. - // Assume the access is atomic. - // Strictly saying even relaxed store cuts off release sequence, - // so must reset the clock. - if (!IsReleaseOrder(mo)) { - NoTsanAtomicStore(a, v, mo); - return; - } - SlotLocker locker(thr); - { - auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); - Lock lock(&s->mtx); - thr->clock.ReleaseStore(&s->clock); - NoTsanAtomicStore(a, v, mo); - } - IncrementEpoch(thr); -} +namespace { template <typename T, T (*F)(volatile T *v, T op)> static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { @@ -303,175 +246,265 @@ static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { return v; } -template<typename T> -static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) { - return func_xchg(a, v); -} +struct OpLoad { + template <typename T> + static T NoTsanAtomic(morder mo, const volatile T *a) { + return atomic_load(to_atomic(a), to_mo(mo)); + } -template<typename T> -static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) { - return func_add(a, v); -} +#if __TSAN_HAS_INT128 && !SANITIZER_GO + static a128 NoTsanAtomic(morder mo, const volatile a128 *a) { + SpinMutexLock lock(&mutex128); + return *a; + } +#endif -template<typename T> -static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) { - return func_sub(a, v); -} + template <typename T> + static T Atomic(ThreadState *thr, uptr pc, morder mo, const volatile T *a) { + DCHECK(IsLoadOrder(mo)); + // This fast-path is critical for performance. + // Assume the access is atomic. + if (!IsAcquireOrder(mo)) { + MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), + kAccessRead | kAccessAtomic); + return NoTsanAtomic(mo, a); + } + // Don't create sync object if it does not exist yet. For example, an atomic + // pointer is initialized to nullptr and then periodically acquire-loaded. + T v = NoTsanAtomic(mo, a); + SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a); + if (s) { + SlotLocker locker(thr); + ReadLock lock(&s->mtx); + thr->clock.Acquire(s->clock); + // Re-read under sync mutex because we need a consistent snapshot + // of the value and the clock we acquire. + v = NoTsanAtomic(mo, a); + } + MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), + kAccessRead | kAccessAtomic); + return v; + } +}; -template<typename T> -static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) { - return func_and(a, v); -} +struct OpStore { + template <typename T> + static void NoTsanAtomic(morder mo, volatile T *a, T v) { + atomic_store(to_atomic(a), v, to_mo(mo)); + } -template<typename T> -static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) { - return func_or(a, v); -} +#if __TSAN_HAS_INT128 && !SANITIZER_GO + static void NoTsanAtomic(morder mo, volatile a128 *a, a128 v) { + SpinMutexLock lock(&mutex128); + *a = v; + } +#endif -template<typename T> -static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) { - return func_xor(a, v); -} + template <typename T> + static void Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) { + DCHECK(IsStoreOrder(mo)); + MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), + kAccessWrite | kAccessAtomic); + // This fast-path is critical for performance. + // Assume the access is atomic. + // Strictly saying even relaxed store cuts off release sequence, + // so must reset the clock. + if (!IsReleaseOrder(mo)) { + NoTsanAtomic(mo, a, v); + return; + } + SlotLocker locker(thr); + { + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); + Lock lock(&s->mtx); + thr->clock.ReleaseStore(&s->clock); + NoTsanAtomic(mo, a, v); + } + IncrementEpoch(thr); + } +}; -template<typename T> -static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) { - return func_nand(a, v); -} +struct OpExchange { + template <typename T> + static T NoTsanAtomic(morder mo, volatile T *a, T v) { + return func_xchg(a, v); + } + template <typename T> + static T Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) { + return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo); + } +}; -template<typename T> -static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo); -} +struct OpFetchAdd { + template <typename T> + static T NoTsanAtomic(morder mo, volatile T *a, T v) { + return func_add(a, v); + } -template<typename T> -static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - return AtomicRMW<T, func_add>(thr, pc, a, v, mo); -} + template <typename T> + static T Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) { + return AtomicRMW<T, func_add>(thr, pc, a, v, mo); + } +}; -template<typename T> -static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - return AtomicRMW<T, func_sub>(thr, pc, a, v, mo); -} +struct OpFetchSub { + template <typename T> + static T NoTsanAtomic(morder mo, volatile T *a, T v) { + return func_sub(a, v); + } -template<typename T> -static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - return AtomicRMW<T, func_and>(thr, pc, a, v, mo); -} + template <typename T> + static T Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) { + return AtomicRMW<T, func_sub>(thr, pc, a, v, mo); + } +}; -template<typename T> -static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - return AtomicRMW<T, func_or>(thr, pc, a, v, mo); -} +struct OpFetchAnd { + template <typename T> + static T NoTsanAtomic(morder mo, volatile T *a, T v) { + return func_and(a, v); + } -template<typename T> -static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - return AtomicRMW<T, func_xor>(thr, pc, a, v, mo); -} + template <typename T> + static T Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) { + return AtomicRMW<T, func_and>(thr, pc, a, v, mo); + } +}; -template<typename T> -static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - return AtomicRMW<T, func_nand>(thr, pc, a, v, mo); -} +struct OpFetchOr { + template <typename T> + static T NoTsanAtomic(morder mo, volatile T *a, T v) { + return func_or(a, v); + } -template<typename T> -static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) { - return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo)); -} + template <typename T> + static T Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) { + return AtomicRMW<T, func_or>(thr, pc, a, v, mo); + } +}; -#if __TSAN_HAS_INT128 -static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v, - morder mo, morder fmo) { - a128 old = *c; - a128 cur = func_cas(a, old, v); - if (cur == old) - return true; - *c = cur; - return false; -} -#endif +struct OpFetchXor { + template <typename T> + static T NoTsanAtomic(morder mo, volatile T *a, T v) { + return func_xor(a, v); + } -template<typename T> -static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) { - NoTsanAtomicCAS(a, &c, v, mo, fmo); - return c; -} + template <typename T> + static T Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) { + return AtomicRMW<T, func_xor>(thr, pc, a, v, mo); + } +}; -template <typename T> -static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v, - morder mo, morder fmo) { - // 31.7.2.18: "The failure argument shall not be memory_order_release - // nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic - // (mo_relaxed) when those are used. - DCHECK(IsLoadOrder(fmo)); +struct OpFetchNand { + template <typename T> + static T NoTsanAtomic(morder mo, volatile T *a, T v) { + return func_nand(a, v); + } - MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic); - if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) { - T cc = *c; - T pr = func_cas(a, cc, v); - if (pr == cc) + template <typename T> + static T Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) { + return AtomicRMW<T, func_nand>(thr, pc, a, v, mo); + } +}; + +struct OpCAS { + template <typename T> + static bool NoTsanAtomic(morder mo, morder fmo, volatile T *a, T *c, T v) { + return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo)); + } + +#if __TSAN_HAS_INT128 + static bool NoTsanAtomic(morder mo, morder fmo, volatile a128 *a, a128 *c, + a128 v) { + a128 old = *c; + a128 cur = func_cas(a, old, v); + if (cur == old) return true; - *c = pr; + *c = cur; return false; } - SlotLocker locker(thr); - bool release = IsReleaseOrder(mo); - bool success; - { - auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); - RWLock lock(&s->mtx, release); - T cc = *c; - T pr = func_cas(a, cc, v); - success = pr == cc; - if (!success) { +#endif + + template <typename T> + static T NoTsanAtomic(morder mo, morder fmo, volatile T *a, T c, T v) { + NoTsanAtomic(mo, fmo, a, &c, v); + return c; + } + + template <typename T> + static bool Atomic(ThreadState *thr, uptr pc, morder mo, morder fmo, + volatile T *a, T *c, T v) { + // 31.7.2.18: "The failure argument shall not be memory_order_release + // nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic + // (mo_relaxed) when those are used. + DCHECK(IsLoadOrder(fmo)); + + MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), + kAccessWrite | kAccessAtomic); + if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) { + T cc = *c; + T pr = func_cas(a, cc, v); + if (pr == cc) + return true; *c = pr; - mo = fmo; + return false; } - if (success && IsAcqRelOrder(mo)) - thr->clock.ReleaseAcquire(&s->clock); - else if (success && IsReleaseOrder(mo)) - thr->clock.Release(&s->clock); - else if (IsAcquireOrder(mo)) - thr->clock.Acquire(s->clock); + SlotLocker locker(thr); + bool release = IsReleaseOrder(mo); + bool success; + { + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); + RWLock lock(&s->mtx, release); + T cc = *c; + T pr = func_cas(a, cc, v); + success = pr == cc; + if (!success) { + *c = pr; + mo = fmo; + } + if (success && IsAcqRelOrder(mo)) + thr->clock.ReleaseAcquire(&s->clock); + else if (success && IsReleaseOrder(mo)) + thr->clock.Release(&s->clock); + else if (IsAcquireOrder(mo)) + thr->clock.Acquire(s->clock); + } + if (success && release) + IncrementEpoch(thr); + return success; } - if (success && release) - IncrementEpoch(thr); - return success; -} -template<typename T> -static T AtomicCAS(ThreadState *thr, uptr pc, - volatile T *a, T c, T v, morder mo, morder fmo) { - AtomicCAS(thr, pc, a, &c, v, mo, fmo); - return c; -} + template <typename T> + static T Atomic(ThreadState *thr, uptr pc, morder mo, morder fmo, + volatile T *a, T c, T v) { + Atomic(thr, pc, mo, fmo, a, &c, v); + return c; + } +}; #if !SANITIZER_GO -static void NoTsanAtomicFence(morder mo) { - __sync_synchronize(); -} +struct OpFence { + static void NoTsanAtomic(morder mo) { __sync_synchronize(); } -static void AtomicFence(ThreadState *thr, uptr pc, morder mo) { - // FIXME(dvyukov): not implemented. - __sync_synchronize(); -} + static void Atomic(ThreadState *thr, uptr pc, morder mo) { + // FIXME(dvyukov): not implemented. + __sync_synchronize(); + } +}; #endif +} // namespace + // Interface functions follow. #if !SANITIZER_GO // C/C++ static morder convert_morder(morder mo) { - if (flags()->force_seq_cst_atomics) - return (morder)mo_seq_cst; + return flags()->force_seq_cst_atomics ? mo_seq_cst : mo; +} +static morder to_morder(int mo) { // Filter out additional memory order flags: // MEMMODEL_SYNC = 1 << 15 // __ATOMIC_HLE_ACQUIRE = 1 << 16 @@ -482,468 +515,481 @@ static morder convert_morder(morder mo) { // since we use __sync_ atomics for actual atomic operations, // we can safely ignore it as well. It also subtly affects semantics, // but we don't model the difference. - return (morder)(mo & 0x7fff); + morder res = static_cast<morder>(static_cast<u8>(mo)); + DCHECK_LE(res, mo_seq_cst); + return res; } -# define ATOMIC_IMPL(func, ...) \ - ThreadState *const thr = cur_thread(); \ - ProcessPendingSignals(thr); \ - if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) \ - return NoTsanAtomic##func(__VA_ARGS__); \ - mo = convert_morder(mo); \ - return Atomic##func(thr, GET_CALLER_PC(), __VA_ARGS__); +template <class Op, class... Types> +ALWAYS_INLINE auto AtomicImpl(morder mo, Types... args) { + ThreadState *const thr = cur_thread(); + ProcessPendingSignals(thr); + if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) + return Op::NoTsanAtomic(mo, args...); + return Op::Atomic(thr, GET_CALLER_PC(), convert_morder(mo), args...); +} extern "C" { SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) { - ATOMIC_IMPL(Load, a, mo); +a8 __tsan_atomic8_load(const volatile a8 *a, int mo) { + return AtomicImpl<OpLoad>(to_morder(mo), a); } SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) { - ATOMIC_IMPL(Load, a, mo); +a16 __tsan_atomic16_load(const volatile a16 *a, int mo) { + return AtomicImpl<OpLoad>(to_morder(mo), a); } SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) { - ATOMIC_IMPL(Load, a, mo); +a32 __tsan_atomic32_load(const volatile a32 *a, int mo) { + return AtomicImpl<OpLoad>(to_morder(mo), a); } SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) { - ATOMIC_IMPL(Load, a, mo); +a64 __tsan_atomic64_load(const volatile a64 *a, int mo) { + return AtomicImpl<OpLoad>(to_morder(mo), a); } -#if __TSAN_HAS_INT128 +# if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) { - ATOMIC_IMPL(Load, a, mo); +a128 __tsan_atomic128_load(const volatile a128 *a, int mo) { + return AtomicImpl<OpLoad>(to_morder(mo), a); } -#endif +# endif SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) { - ATOMIC_IMPL(Store, a, v, mo); +void __tsan_atomic8_store(volatile a8 *a, a8 v, int mo) { + return AtomicImpl<OpStore>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) { - ATOMIC_IMPL(Store, a, v, mo); +void __tsan_atomic16_store(volatile a16 *a, a16 v, int mo) { + return AtomicImpl<OpStore>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) { - ATOMIC_IMPL(Store, a, v, mo); +void __tsan_atomic32_store(volatile a32 *a, a32 v, int mo) { + return AtomicImpl<OpStore>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) { - ATOMIC_IMPL(Store, a, v, mo); +void __tsan_atomic64_store(volatile a64 *a, a64 v, int mo) { + return AtomicImpl<OpStore>(to_morder(mo), a, v); } -#if __TSAN_HAS_INT128 +# if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) { - ATOMIC_IMPL(Store, a, v, mo); +void __tsan_atomic128_store(volatile a128 *a, a128 v, int mo) { + return AtomicImpl<OpStore>(to_morder(mo), a, v); } -#endif +# endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) { - ATOMIC_IMPL(Exchange, a, v, mo); +a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, int mo) { + return AtomicImpl<OpExchange>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) { - ATOMIC_IMPL(Exchange, a, v, mo); +a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, int mo) { + return AtomicImpl<OpExchange>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) { - ATOMIC_IMPL(Exchange, a, v, mo); +a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, int mo) { + return AtomicImpl<OpExchange>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) { - ATOMIC_IMPL(Exchange, a, v, mo); +a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, int mo) { + return AtomicImpl<OpExchange>(to_morder(mo), a, v); } -#if __TSAN_HAS_INT128 +# if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) { - ATOMIC_IMPL(Exchange, a, v, mo); +a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, int mo) { + return AtomicImpl<OpExchange>(to_morder(mo), a, v); } -#endif +# endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) { - ATOMIC_IMPL(FetchAdd, a, v, mo); +a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, int mo) { + return AtomicImpl<OpFetchAdd>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) { - ATOMIC_IMPL(FetchAdd, a, v, mo); +a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, int mo) { + return AtomicImpl<OpFetchAdd>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) { - ATOMIC_IMPL(FetchAdd, a, v, mo); +a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, int mo) { + return AtomicImpl<OpFetchAdd>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) { - ATOMIC_IMPL(FetchAdd, a, v, mo); +a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, int mo) { + return AtomicImpl<OpFetchAdd>(to_morder(mo), a, v); } -#if __TSAN_HAS_INT128 +# if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) { - ATOMIC_IMPL(FetchAdd, a, v, mo); +a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, int mo) { + return AtomicImpl<OpFetchAdd>(to_morder(mo), a, v); } -#endif +# endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) { - ATOMIC_IMPL(FetchSub, a, v, mo); +a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, int mo) { + return AtomicImpl<OpFetchSub>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) { - ATOMIC_IMPL(FetchSub, a, v, mo); +a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, int mo) { + return AtomicImpl<OpFetchSub>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) { - ATOMIC_IMPL(FetchSub, a, v, mo); +a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, int mo) { + return AtomicImpl<OpFetchSub>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) { - ATOMIC_IMPL(FetchSub, a, v, mo); +a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, int mo) { + return AtomicImpl<OpFetchSub>(to_morder(mo), a, v); } -#if __TSAN_HAS_INT128 +# if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) { - ATOMIC_IMPL(FetchSub, a, v, mo); +a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, int mo) { + return AtomicImpl<OpFetchSub>(to_morder(mo), a, v); } -#endif +# endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) { - ATOMIC_IMPL(FetchAnd, a, v, mo); +a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, int mo) { + return AtomicImpl<OpFetchAnd>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) { - ATOMIC_IMPL(FetchAnd, a, v, mo); +a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, int mo) { + return AtomicImpl<OpFetchAnd>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) { - ATOMIC_IMPL(FetchAnd, a, v, mo); +a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, int mo) { + return AtomicImpl<OpFetchAnd>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) { - ATOMIC_IMPL(FetchAnd, a, v, mo); +a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, int mo) { + return AtomicImpl<OpFetchAnd>(to_morder(mo), a, v); } -#if __TSAN_HAS_INT128 +# if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) { - ATOMIC_IMPL(FetchAnd, a, v, mo); +a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, int mo) { + return AtomicImpl<OpFetchAnd>(to_morder(mo), a, v); } -#endif +# endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) { - ATOMIC_IMPL(FetchOr, a, v, mo); +a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, int mo) { + return AtomicImpl<OpFetchOr>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) { - ATOMIC_IMPL(FetchOr, a, v, mo); +a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, int mo) { + return AtomicImpl<OpFetchOr>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) { - ATOMIC_IMPL(FetchOr, a, v, mo); +a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, int mo) { + return AtomicImpl<OpFetchOr>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) { - ATOMIC_IMPL(FetchOr, a, v, mo); +a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, int mo) { + return AtomicImpl<OpFetchOr>(to_morder(mo), a, v); } -#if __TSAN_HAS_INT128 +# if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) { - ATOMIC_IMPL(FetchOr, a, v, mo); +a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, int mo) { + return AtomicImpl<OpFetchOr>(to_morder(mo), a, v); } -#endif +# endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) { - ATOMIC_IMPL(FetchXor, a, v, mo); +a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, int mo) { + return AtomicImpl<OpFetchXor>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) { - ATOMIC_IMPL(FetchXor, a, v, mo); +a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, int mo) { + return AtomicImpl<OpFetchXor>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) { - ATOMIC_IMPL(FetchXor, a, v, mo); +a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, int mo) { + return AtomicImpl<OpFetchXor>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) { - ATOMIC_IMPL(FetchXor, a, v, mo); +a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, int mo) { + return AtomicImpl<OpFetchXor>(to_morder(mo), a, v); } -#if __TSAN_HAS_INT128 +# if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) { - ATOMIC_IMPL(FetchXor, a, v, mo); +a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, int mo) { + return AtomicImpl<OpFetchXor>(to_morder(mo), a, v); } -#endif +# endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) { - ATOMIC_IMPL(FetchNand, a, v, mo); +a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, int mo) { + return AtomicImpl<OpFetchNand>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) { - ATOMIC_IMPL(FetchNand, a, v, mo); +a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, int mo) { + return AtomicImpl<OpFetchNand>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) { - ATOMIC_IMPL(FetchNand, a, v, mo); +a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, int mo) { + return AtomicImpl<OpFetchNand>(to_morder(mo), a, v); } SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) { - ATOMIC_IMPL(FetchNand, a, v, mo); +a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, int mo) { + return AtomicImpl<OpFetchNand>(to_morder(mo), a, v); } -#if __TSAN_HAS_INT128 +# if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE -a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) { - ATOMIC_IMPL(FetchNand, a, v, mo); +a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, int mo) { + return AtomicImpl<OpFetchNand>(to_morder(mo), a, v); } -#endif +# endif SANITIZER_INTERFACE_ATTRIBUTE -int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, - morder mo, morder fmo) { - ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, int mo, + int fmo) { + return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v); } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v, - morder mo, morder fmo) { - ATOMIC_IMPL(CAS, a, c, v, mo, fmo); + int mo, int fmo) { + return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v); } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v, - morder mo, morder fmo) { - ATOMIC_IMPL(CAS, a, c, v, mo, fmo); + int mo, int fmo) { + return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v); } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v, - morder mo, morder fmo) { - ATOMIC_IMPL(CAS, a, c, v, mo, fmo); + int mo, int fmo) { + return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v); } -#if __TSAN_HAS_INT128 +# if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v, - morder mo, morder fmo) { - ATOMIC_IMPL(CAS, a, c, v, mo, fmo); + int mo, int fmo) { + return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v); } -#endif +# endif SANITIZER_INTERFACE_ATTRIBUTE -int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, - morder mo, morder fmo) { - ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, int mo, + int fmo) { + return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v); } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v, - morder mo, morder fmo) { - ATOMIC_IMPL(CAS, a, c, v, mo, fmo); + int mo, int fmo) { + return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v); } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v, - morder mo, morder fmo) { - ATOMIC_IMPL(CAS, a, c, v, mo, fmo); + int mo, int fmo) { + return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v); } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v, - morder mo, morder fmo) { - ATOMIC_IMPL(CAS, a, c, v, mo, fmo); + int mo, int fmo) { + return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v); } -#if __TSAN_HAS_INT128 +# if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v, - morder mo, morder fmo) { - ATOMIC_IMPL(CAS, a, c, v, mo, fmo); + int mo, int fmo) { + return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v); } -#endif +# endif SANITIZER_INTERFACE_ATTRIBUTE -a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, - morder mo, morder fmo) { - ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, int mo, + int fmo) { + return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v); } SANITIZER_INTERFACE_ATTRIBUTE -a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, - morder mo, morder fmo) { - ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, int mo, + int fmo) { + return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v); } SANITIZER_INTERFACE_ATTRIBUTE -a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, - morder mo, morder fmo) { - ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, int mo, + int fmo) { + return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v); } SANITIZER_INTERFACE_ATTRIBUTE -a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, - morder mo, morder fmo) { - ATOMIC_IMPL(CAS, a, c, v, mo, fmo); +a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, int mo, + int fmo) { + return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v); } -#if __TSAN_HAS_INT128 +# if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v, - morder mo, morder fmo) { - ATOMIC_IMPL(CAS, a, c, v, mo, fmo); + int mo, int fmo) { + return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v); } -#endif +# endif SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_atomic_thread_fence(morder mo) { ATOMIC_IMPL(Fence, mo); } +void __tsan_atomic_thread_fence(int mo) { + return AtomicImpl<OpFence>(to_morder(mo)); +} SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_atomic_signal_fence(morder mo) { -} +void __tsan_atomic_signal_fence(int mo) {} } // extern "C" #else // #if !SANITIZER_GO // Go -# define ATOMIC(func, ...) \ - if (thr->ignore_sync) { \ - NoTsanAtomic##func(__VA_ARGS__); \ - } else { \ - FuncEntry(thr, cpc); \ - Atomic##func(thr, pc, __VA_ARGS__); \ - FuncExit(thr); \ - } +template <class Op, class... Types> +void AtomicGo(ThreadState *thr, uptr cpc, uptr pc, Types... args) { + if (thr->ignore_sync) { + (void)Op::NoTsanAtomic(args...); + } else { + FuncEntry(thr, cpc); + (void)Op::Atomic(thr, pc, args...); + FuncExit(thr); + } +} -# define ATOMIC_RET(func, ret, ...) \ - if (thr->ignore_sync) { \ - (ret) = NoTsanAtomic##func(__VA_ARGS__); \ - } else { \ - FuncEntry(thr, cpc); \ - (ret) = Atomic##func(thr, pc, __VA_ARGS__); \ - FuncExit(thr); \ - } +template <class Op, class... Types> +auto AtomicGoRet(ThreadState *thr, uptr cpc, uptr pc, Types... args) { + if (thr->ignore_sync) { + return Op::NoTsanAtomic(args...); + } else { + FuncEntry(thr, cpc); + auto ret = Op::Atomic(thr, pc, args...); + FuncExit(thr); + return ret; + } +} extern "C" { SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { - ATOMIC_RET(Load, *(a32*)(a+8), *(a32**)a, mo_acquire); + *(a32 *)(a + 8) = AtomicGoRet<OpLoad>(thr, cpc, pc, mo_acquire, *(a32 **)a); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { - ATOMIC_RET(Load, *(a64*)(a+8), *(a64**)a, mo_acquire); + *(a64 *)(a + 8) = AtomicGoRet<OpLoad>(thr, cpc, pc, mo_acquire, *(a64 **)a); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { - ATOMIC(Store, *(a32**)a, *(a32*)(a+8), mo_release); + AtomicGo<OpStore>(thr, cpc, pc, mo_release, *(a32 **)a, *(a32 *)(a + 8)); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { - ATOMIC(Store, *(a64**)a, *(a64*)(a+8), mo_release); + AtomicGo<OpStore>(thr, cpc, pc, mo_release, *(a64 **)a, *(a64 *)(a + 8)); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { - ATOMIC_RET(FetchAdd, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); + *(a32 *)(a + 16) = AtomicGoRet<OpFetchAdd>(thr, cpc, pc, mo_acq_rel, + *(a32 **)a, *(a32 *)(a + 8)); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { - ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); + *(a64 *)(a + 16) = AtomicGoRet<OpFetchAdd>(thr, cpc, pc, mo_acq_rel, + *(a64 **)a, *(a64 *)(a + 8)); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { - ATOMIC_RET(FetchAnd, *(a32 *)(a + 16), *(a32 **)a, *(a32 *)(a + 8), - mo_acq_rel); + *(a32 *)(a + 16) = AtomicGoRet<OpFetchAnd>(thr, cpc, pc, mo_acq_rel, + *(a32 **)a, *(a32 *)(a + 8)); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { - ATOMIC_RET(FetchAnd, *(a64 *)(a + 16), *(a64 **)a, *(a64 *)(a + 8), - mo_acq_rel); + *(a64 *)(a + 16) = AtomicGoRet<OpFetchAnd>(thr, cpc, pc, mo_acq_rel, + *(a64 **)a, *(a64 *)(a + 8)); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { - ATOMIC_RET(FetchOr, *(a32 *)(a + 16), *(a32 **)a, *(a32 *)(a + 8), - mo_acq_rel); + *(a32 *)(a + 16) = AtomicGoRet<OpFetchOr>(thr, cpc, pc, mo_acq_rel, + *(a32 **)a, *(a32 *)(a + 8)); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { - ATOMIC_RET(FetchOr, *(a64 *)(a + 16), *(a64 **)a, *(a64 *)(a + 8), - mo_acq_rel); + *(a64 *)(a + 16) = AtomicGoRet<OpFetchOr>(thr, cpc, pc, mo_acq_rel, + *(a64 **)a, *(a64 *)(a + 8)); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { - ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); + *(a32 *)(a + 16) = AtomicGoRet<OpExchange>(thr, cpc, pc, mo_acq_rel, + *(a32 **)a, *(a32 *)(a + 8)); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { - ATOMIC_RET(Exchange, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); + *(a64 *)(a + 16) = AtomicGoRet<OpExchange>(thr, cpc, pc, mo_acq_rel, + *(a64 **)a, *(a64 *)(a + 8)); } SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_go_atomic32_compare_exchange( - ThreadState *thr, uptr cpc, uptr pc, u8 *a) { - a32 cur = 0; - a32 cmp = *(a32*)(a+8); - ATOMIC_RET(CAS, cur, *(a32**)a, cmp, *(a32*)(a+12), mo_acq_rel, mo_acquire); - *(bool*)(a+16) = (cur == cmp); +void __tsan_go_atomic32_compare_exchange(ThreadState *thr, uptr cpc, uptr pc, + u8 *a) { + a32 cmp = *(a32 *)(a + 8); + a32 cur = AtomicGoRet<OpCAS>(thr, cpc, pc, mo_acq_rel, mo_acquire, *(a32 **)a, + cmp, *(a32 *)(a + 12)); + *(bool *)(a + 16) = (cur == cmp); } SANITIZER_INTERFACE_ATTRIBUTE -void __tsan_go_atomic64_compare_exchange( - ThreadState *thr, uptr cpc, uptr pc, u8 *a) { - a64 cur = 0; - a64 cmp = *(a64*)(a+8); - ATOMIC_RET(CAS, cur, *(a64**)a, cmp, *(a64*)(a+16), mo_acq_rel, mo_acquire); - *(bool*)(a+24) = (cur == cmp); +void __tsan_go_atomic64_compare_exchange(ThreadState *thr, uptr cpc, uptr pc, + u8 *a) { + a64 cmp = *(a64 *)(a + 8); + a64 cur = AtomicGoRet<OpCAS>(thr, cpc, pc, mo_acq_rel, mo_acquire, *(a64 **)a, + cmp, *(a64 *)(a + 16)); + *(bool *)(a + 24) = (cur == cmp); } } // extern "C" #endif // #if !SANITIZER_GO diff --git a/lib/tsan/tsan_mman.cpp b/lib/tsan/tsan_mman.cpp @@ -252,7 +252,7 @@ void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr size, uptr n) { if (AllocatorMayReturnNull()) return SetErrnoOnNull(nullptr); GET_STACK_TRACE_FATAL(thr, pc); - ReportReallocArrayOverflow(size, n, &stack); + ReportReallocArrayOverflow(n, size, &stack); } return user_realloc(thr, pc, p, size * n); } diff --git a/lib/tsan/tsan_platform_linux.cpp b/lib/tsan/tsan_platform_linux.cpp @@ -418,7 +418,6 @@ void InitializePlatform() { Die(); } - InitTlsSize(); #endif // !SANITIZER_GO } diff --git a/lib/tsan/tsan_rtl.cpp b/lib/tsan/tsan_rtl.cpp @@ -673,7 +673,8 @@ void CheckUnwind() { thr->ignore_reads_and_writes++; atomic_store_relaxed(&thr->in_signal_handler, 0); #endif - PrintCurrentStackSlow(StackTrace::GetCurrentPc()); + PrintCurrentStack(StackTrace::GetCurrentPc(), + common_flags()->fast_unwind_on_fatal); } bool is_initialized; @@ -806,6 +807,7 @@ int Finalize(ThreadState *thr) { #if !SANITIZER_GO void ForkBefore(ThreadState* thr, uptr pc) SANITIZER_NO_THREAD_SAFETY_ANALYSIS { + VReport(2, "BeforeFork tid: %llu\n", GetTid()); GlobalProcessorLock(); // Detaching from the slot makes OnUserFree skip writing to the shadow. // The slot will be locked so any attempts to use it will deadlock anyway. @@ -847,6 +849,7 @@ static void ForkAfter(ThreadState* thr, SlotAttachAndLock(thr); SlotUnlock(thr); GlobalProcessorUnlock(); + VReport(2, "AfterFork tid: %llu\n", GetTid()); } void ForkParentAfter(ThreadState* thr, uptr pc) { ForkAfter(thr, false); } diff --git a/lib/tsan/tsan_rtl.h b/lib/tsan/tsan_rtl.h @@ -514,7 +514,7 @@ bool IsExpectedReport(uptr addr, uptr size); StackID CurrentStackId(ThreadState *thr, uptr pc); ReportStack *SymbolizeStackId(StackID stack_id); void PrintCurrentStack(ThreadState *thr, uptr pc); -void PrintCurrentStackSlow(uptr pc); // uses libunwind +void PrintCurrentStack(uptr pc, bool fast); // may uses libunwind MBlock *JavaHeapBlock(uptr addr, uptr *start); void Initialize(ThreadState *thr); diff --git a/lib/tsan/tsan_rtl_report.cpp b/lib/tsan/tsan_rtl_report.cpp @@ -828,18 +828,18 @@ void PrintCurrentStack(ThreadState *thr, uptr pc) { PrintStack(SymbolizeStack(trace)); } -// Always inlining PrintCurrentStackSlow, because LocatePcInTrace assumes +// Always inlining PrintCurrentStack, because LocatePcInTrace assumes // __sanitizer_print_stack_trace exists in the actual unwinded stack, but -// tail-call to PrintCurrentStackSlow breaks this assumption because +// tail-call to PrintCurrentStack breaks this assumption because // __sanitizer_print_stack_trace disappears after tail-call. // However, this solution is not reliable enough, please see dvyukov's comment // http://reviews.llvm.org/D19148#406208 // Also see PR27280 comment 2 and 3 for breaking examples and analysis. -ALWAYS_INLINE USED void PrintCurrentStackSlow(uptr pc) { +ALWAYS_INLINE USED void PrintCurrentStack(uptr pc, bool fast) { #if !SANITIZER_GO uptr bp = GET_CURRENT_FRAME(); auto *ptrace = New<BufferedStackTrace>(); - ptrace->Unwind(pc, bp, nullptr, false); + ptrace->Unwind(pc, bp, nullptr, fast); for (uptr i = 0; i < ptrace->size / 2; i++) { uptr tmp = ptrace->trace_buffer[i]; @@ -857,6 +857,6 @@ using namespace __tsan; extern "C" { SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_print_stack_trace() { - PrintCurrentStackSlow(StackTrace::GetCurrentPc()); + PrintCurrentStack(StackTrace::GetCurrentPc(), false); } } // extern "C" diff --git a/lib/tsan/tsan_rtl_thread.cpp b/lib/tsan/tsan_rtl_thread.cpp @@ -165,14 +165,16 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id, #endif uptr stk_addr = 0; - uptr stk_size = 0; + uptr stk_end = 0; uptr tls_addr = 0; - uptr tls_size = 0; + uptr tls_end = 0; #if !SANITIZER_GO if (thread_type != ThreadType::Fiber) - GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_size, &tls_addr, - &tls_size); + GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_end, &tls_addr, + &tls_end); #endif + uptr stk_size = stk_end - stk_addr; + uptr tls_size = tls_end - tls_addr; thr->stk_addr = stk_addr; thr->stk_size = stk_size; thr->tls_addr = tls_addr; diff --git a/lib/tsan/tsan_spinlock_defs_mac.h b/lib/tsan/tsan_spinlock_defs_mac.h @@ -1,45 +0,0 @@ -//===-- tsan_spinlock_defs_mac.h -------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file is a part of ThreadSanitizer (TSan), a race detector. -// -// Mac-specific forward-declared function defintions that may be -// deprecated in later versions of the OS. -// These are needed for interceptors. -// -//===----------------------------------------------------------------------===// - -#if SANITIZER_APPLE - -#ifndef TSAN_SPINLOCK_DEFS_MAC_H -#define TSAN_SPINLOCK_DEFS_MAC_H - -#include <stdint.h> - -extern "C" { - -/* -Provides forward declarations related to OSSpinLocks on Darwin. These functions are -deprecated on macOS version 10.12 and later, -and are no longer included in the system headers. - -However, the symbols are still available on the system, so we provide these forward -declarations to prevent compilation errors in tsan_interceptors_mac.cpp, which -references these functions when defining TSAN interceptor functions. -*/ - -typedef int32_t OSSpinLock; - -void OSSpinLockLock(volatile OSSpinLock *__lock); -void OSSpinLockUnlock(volatile OSSpinLock *__lock); -bool OSSpinLockTry(volatile OSSpinLock *__lock); - -} - -#endif //TSAN_SPINLOCK_DEFS_MAC_H -#endif // SANITIZER_APPLE diff --git a/src/libtsan.zig b/src/libtsan.zig @@ -410,9 +410,6 @@ const sanitizer_common_sources = [_][]const u8{ "sanitizer_allocator.cpp", "sanitizer_chained_origin_depot.cpp", "sanitizer_common.cpp", - "sanitizer_coverage_win_dll_thunk.cpp", - "sanitizer_coverage_win_dynamic_runtime_thunk.cpp", - "sanitizer_coverage_win_weak_interception.cpp", "sanitizer_deadlock_detector1.cpp", "sanitizer_deadlock_detector2.cpp", "sanitizer_errno.cpp", @@ -452,9 +449,7 @@ const sanitizer_common_sources = [_][]const u8{ "sanitizer_tls_get_addr.cpp", "sanitizer_type_traits.cpp", "sanitizer_win.cpp", - "sanitizer_win_dll_thunk.cpp", - "sanitizer_win_dynamic_runtime_thunk.cpp", - "sanitizer_win_weak_interception.cpp", + "sanitizer_win_interception.cpp", }; const sanitizer_nolibc_sources = [_][]const u8{ @@ -490,6 +485,7 @@ const sanitizer_symbolizer_sources = [_][]const u8{ "sanitizer_symbolizer_report.cpp", "sanitizer_symbolizer_report_fuchsia.cpp", "sanitizer_symbolizer_win.cpp", + "sanitizer_thread_history.cpp", "sanitizer_unwind_linux_libcdep.cpp", "sanitizer_unwind_fuchsia.cpp", "sanitizer_unwind_win.cpp",