Early segfaults in __tsan::TraceAddEvent

[djdeath]

Hi,

I'm trying to hunt down a race condition in Mesa3D.
I've compiled Mesa & a test program successfully with -fsanitize=address, but didn't get much insight into where the problem might be.
I'm now trying -fsanitize=thread but I'm running into a segfault pretty early in the initialization of the program.

Here is the backtrace :

Program received signal SIGSEGV, Segmentation fault.
0x00007ffff6da68f7 in __tsan::TraceAddEvent (addr=140737334598989, typ=__tsan::EventTypeFuncEnter, fs=..., thr=)
at ../../../../src/libsanitizer/tsan/tsan_rtl.h:723
723 ../../../../src/libsanitizer/tsan/tsan_rtl.h: No such file or directory.
(gdb) bt
#0 0x00007ffff6da68f7 in __tsan::TraceAddEvent (addr=140737334598989, typ=__tsan::EventTypeFuncEnter, fs=..., thr=)
at ../../../../src/libsanitizer/tsan/tsan_rtl.h:723
#1 __tsan::FuncEntry (pc=140737334598989, thr=) at ../../../../src/libsanitizer/tsan/tsan_rtl.cc:913
#2 __tsan_func_entry (pc=0x7ffff6d5dd4d <__interceptor_mmap(void*, long_t, int, int, int, unsigned int)+173>)
at ../../../../src/libsanitizer/tsan/tsan_interface_inl.h:70
#3 0x00007ffff6b34641 in mmap (addr=0x0, len=4096, prot=3, flags=34, fildes=-1, off=0) at intel_stub.c:161
#4 0x00007ffff6d5dd4d in __interceptor_mmap (addr=addr@entry=0x0, sz=sz@entry=4096, prot=prot@entry=3, flags=flags@entry=34, fd=fd@entry=-1,
off=off@entry=0) at ../../../../src/libsanitizer/tsan/tsan_interceptors.cc:734
#5 0x00007ffff6dd082b in __asan_backtrace_alloc (state=state@entry=0x7fffffffd430, size=size@entry=72, error_callback=error_callback@entry=
0x7ffff6dbdcc0 <__sanitizer::(anonymous namespace)::ErrorCallback(void*, char const*, int)>, data=data@entry=0x0)
at ../../../../src/libsanitizer/libbacktrace/../../libbacktrace/mmap.c:140
#6 0x00007ffff6dc5ac3 in __asan_backtrace_create_state (filename=filename@entry=0x7ffff6dd7ec5 "/proc/self/exe", threaded=threaded@entry=0,
error_callback=error_callback@entry=0x7ffff6dbdcc0 <__sanitizer::(anonymous namespace)::ErrorCallback(void*, char const*, int)>, data=data@entry=0x0)
at ../../../../src/libsanitizer/libbacktrace/../../libbacktrace/state.c:65
#7 0x00007ffff6dbdda0 in __sanitizer::LibbacktraceSymbolizer::get (alloc=alloc@entry=0x7ffff7dd6760 <_sanitizer::Symbolizer::symbolizer_allocator>)
at ../../../../src/libsanitizer/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc:143
#8 0x00007ffff6dbe30f in __sanitizer::Symbolizer::PlatformInit () at ../../../../src/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc:734
#9 0x00007ffff6dbe185 in __sanitizer::Symbolizer::GetOrInit () at ../../../../src/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cc:21
#10 0x00007ffff6da1062 in __tsan::Initialize (thr=thr@entry=0x7ffff7f618c0) at ../../../../src/libsanitizer/tsan/tsan_rtl.cc:326
#11 0x00007ffff6d5bb98 in ScopedInterceptor::ScopedInterceptor (this=0x7fffffffd5c0, thr=0x7ffff7f618c0, fname=, pc=140737250184790)
at ../../../../src/libsanitizer/tsan/tsan_interceptors.cc:190
#12 0x00007ffff6d5c0ce in __interceptor___cxa_atexit (f=0x7ffff1cde4d0, arg=0x7ffff1fd0c90, dso=0x7ffff1fd0940)
at ../../../../src/libsanitizer/tsan/tsan_interceptors.cc:321
#13 0x00007ffff1cdce56 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#14 0x00007ffff7de74ea in call_init (l=, argc=argc@entry=2, argv=argv@entry=0x7fffffffd698, env=env@entry=0x7fffffffd6b0) at dl-init.c:72
#15 0x00007ffff7de75fb in call_init (env=0x7fffffffd6b0, argv=0x7fffffffd698, argc=2, l=) at dl-init.c:30
#16 _dl_init (main_map=0x7ffff7ffe168, argc=2, argv=0x7fffffffd698, env=0x7fffffffd6b0) at dl-init.c:120
#17 0x00007ffff7dd7cfa in _dl_start_user () from /lib64/ld-linux-x86-64.so.2
#18 0x0000000000000002 in ?? ()
#19 0x00007fffffffd9f0 in ?? ()
#20 0x00007fffffffda15 in ?? ()
#21 0x0000000000000000 in ?? ()

Disassembly :

Dump of assembler code for function __tsan_func_entry(void*):
0x00007ffff6da6880 <+0>: push %rbx
0x00007ffff6da6881 <+1>: mov %rdi,%rbx
0x00007ffff6da6884 <+4>: mov %fs:0x0,%rax
0x00007ffff6da688d <+13>: add 0x253bbc(%rip),%rax # 0x7ffff6ffa450
0x00007ffff6da6894 <+20>: mov (%rax),%rsi
0x00007ffff6da6897 <+23>: lea 0x1(%rsi),%rdx
0x00007ffff6da689b <+27>: mov %rdx,%rcx
0x00007ffff6da689e <+30>: mov %rdx,(%rax)
0x00007ffff6da68a1 <+33>: mov $0x1,%eax
0x00007ffff6da68a6 <+38>: shr $0x2a,%rcx
0x00007ffff6da68aa <+42>: and $0x7,%ecx
0x00007ffff6da68ad <+45>: add $0xf,%ecx
0x00007ffff6da68b0 <+48>: shl %cl,%rax
0x00007ffff6da68b3 <+51>: lea -0x1(%rax),%rcx
0x00007ffff6da68b7 <+55>: movabs $0x3ffffffffff,%rax
0x00007ffff6da68c1 <+65>: and %rdx,%rax
0x00007ffff6da68c4 <+68>: and %rcx,%rax
0x00007ffff6da68c7 <+71>: test $0x3fff,%eax
0x00007ffff6da68cc <+76>: je 0x7ffff6da6928 <__tsan_func_entry(void*)+168>
0x00007ffff6da68ce <+78>: add %rdx,%rdx
0x00007ffff6da68d1 <+81>: shr $0x33,%rdx
0x00007ffff6da68d5 <+85>: imul $0x2130000,%rdx,%rdx
0x00007ffff6da68dc <+92>: lea (%rdx,%rax,8),%rcx
0x00007ffff6da68e0 <+96>: movabs $0x2000000000000000,%rax
0x00007ffff6da68ea <+106>: movabs $0x600000000000,%rdx
0x00007ffff6da68f4 <+116>: or %rbx,%rax
=> 0x00007ffff6da68f7 <+119>: mov %rax,(%rcx,%rdx,1)
0x00007ffff6da68fb <+123>: mov %fs:0x0,%rax
0x00007ffff6da6904 <+132>: add 0x253b45(%rip),%rax # 0x7ffff6ffa450
0x00007ffff6da690b <+139>: mov 0xb8(%rax),%rdx
0x00007ffff6da6912 <+146>: mov %rbx,(%rdx)
0x00007ffff6da6915 <+149>: add $0x8,%rdx
0x00007ffff6da6919 <+153>: mov %rdx,0xb8(%rax)
0x00007ffff6da6920 <+160>: pop %rbx
0x00007ffff6da6921 <+161>: retq
0x00007ffff6da6922 <+162>: nopw 0x0(%rax,%rax,1)
0x00007ffff6da6928 <+168>: sub $0x400,%rsp
0x00007ffff6da692f <+175>: callq 0x7ffff6daf9b7 <__tsan_trace_switch_thunk>
0x00007ffff6da6934 <+180>: add $0x400,%rsp
0x00007ffff6da693b <+187>: jmp 0x7ffff6da68ce <__tsan_func_entry(void*)+78>
End of assembler dump.

Registers :

rax 0x20007ffff6d5dd4d 2305983746548292941
rbx 0x7ffff6d5dd4d 140737334598989
rcx 0x8 8
rdx 0x600000000000 105553116266496
rsi 0x0 0
rdi 0x7ffff6d5dd4d 140737334598989
rbp 0x7fffffffd360 0x7fffffffd360
rsp 0x7fffffffd310 0x7fffffffd310
r8 0xffffffff 4294967295
r9 0x0 0
r10 0x531 1329
r11 0x7ffff6da6880 140737334896768
r12 0x1000 4096
r13 0x22 34
r14 0x7ffff7fca9c0 140737353918912
r15 0x7ffff7f618c0 140737353488576
rip 0x7ffff6da68f7 0x7ffff6da68f7 <__tsan_func_entry(void*)+119>
eflags 0x10206 [ PF IF RF ]
cs 0x33 51
ss 0x2b 43
ds 0x0 0
es 0x0 0
fs 0x0 0
gs 0x0 0

This is on Ubuntu 16.04.2 and the compiler is gcc 5.4.0.

[dvyukov]

I don't see any issues that mention LibbacktraceSymbolizer here nor in gcc tracker...
But this is definitely an issue with gcc unwinder.
Can you try with newer gcc or clang?
If it happens with latest gcc we need to file a bug at https://gcc.gnu.org/bugzilla

[djdeath]

Right, sorry, maybe that's because I left ASAN_SYMBOLIZER_PATH set to an llvm-symbolizer.
Here is the updated backtrace with that disabled :

Program received signal SIGSEGV, Segmentation fault.
0x0000000000000000 in ?? ()
(gdb) bt
#0 0x0000000000000000 in ?? ()
#1 0x00007ffff3a4b152 in __tsan::InitializeInterceptors () at ../../../../src/libsanitizer/tsan/tsan_interceptors.cc:2552
#2 0x00007ffff3a5fea3 in __tsan::Initialize (thr=thr@entry=0x7ffff7f60800) at ../../../../src/libsanitizer/tsan/tsan_rtl.cc:313
#3 0x00007ffff3a1ab98 in ScopedInterceptor::ScopedInterceptor (this=0x7fffffffd5f0, thr=0x7ffff7f60800, fname=, pc=140737236360280)
at ../../../../src/libsanitizer/tsan/tsan_interceptors.cc:190
#4 0x00007ffff3a1d391 in __cxa_guard_acquire (g=0x7ffff1277cc8) at ../../../../src/libsanitizer/tsan/tsan_interceptors.cc:811
#5 0x00007ffff0fadc58 in std::future_category() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#6 0x00007ffff0f80e99 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#7 0x00007ffff7de74ea in call_init (l=, argc=argc@entry=2, argv=argv@entry=0x7fffffffd6e8, env=env@entry=0x7fffffffd700) at dl-init.c:72
#8 0x00007ffff7de75fb in call_init (env=0x7fffffffd700, argv=0x7fffffffd6e8, argc=2, l=) at dl-init.c:30
#9 _dl_init (main_map=0x7ffff7ffe168, argc=2, argv=0x7fffffffd6e8, env=0x7fffffffd700) at dl-init.c:120
#10 0x00007ffff7dd7cfa in _dl_start_user () from /lib64/ld-linux-x86-64.so.2
#11 0x0000000000000002 in ?? ()
#12 0x00007fffffffda37 in ?? ()
#13 0x00007fffffffda5c in ?? ()
#14 0x0000000000000000 in ?? ()

Registers :

rax 0x7ffff3a18b60 140737280838496
rbx 0x7ffff59e3aa0 140737314175648
rcx 0x1 1
rdx 0x0 0
rsi 0x0 0
rdi 0x7ffff3a194e0 140737280840928
rbp 0x7ffff59e3aa0 0x7ffff59e3aa0 <__interceptor__exit(int)>
rsp 0x7fffffffd538 0x7fffffffd538
r8 0x0 0
r9 0x0 0
r10 0x7ffff7ffd040 140737354125376
r11 0x7ffff0efbfec 140737235632108
r12 0x7fffffffd5f0 140737488344560
r13 0x7fffffffd5f0 140737488344560
r14 0x7ffff7fc9a40 140737353914944
r15 0x608d90 6327696
rip 0x0 0x0
eflags 0x10246 [ PF ZF IF RF ]
cs 0x33 51
ss 0x2b 43
ds 0x0 0
es 0x0 0
fs 0x0 0
gs 0x0 0

[dvyukov]

Enabling both asan and tsan at the same time is not supported. Enable them separately.