Skip to content

Spinning thread taking 100% cpu in http/2 dependency tree #3990

@bryancall

Description

@bryancall
  47.61%  traffic_server         [.] Http2DependencyTree::Tree<Http2Stream*>::update
  46.37%  traffic_server         [.] PriorityQueue<Http2DependencyTree::Node*, PriorityQueueLess<Http2DependencyTree::Node*> >::_bubble_down
   0.28%  [kernel]               [k] vsnprintf
   0.27%  [kernel]               [k] format_decode


-   84.36%    43.15%  traffic_server         [.] Http2DependencyTree::Tree<Http2Stream*>::update                                                                                                                                             ▒
   + 43.15% start_thread                                                                                                                                                                                                                     ▒
   - 41.20% Http2DependencyTree::Tree<Http2Stream*>::update                                                                                                                                                                                  ▒
        PriorityQueue<Http2DependencyTree::Node*, PriorityQueueLess<Http2DependencyTree::Node*> >::_bubble_down



root@qa1 262/0 # pstack $(pidof traffic_server)
Thread 23 (Thread 0x2b3a3592a700 (LWP 14836)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000e4697e in ink_cond_timedwait (t=0x2b3a35929ad0, mp=0x2b3a35203330, cp=0x2b3a35203358) at ../../lib/ts/ink_thread.h:259
#2  ProtectedQueue::wait (this=0x2b3a35203310, timeout=<optimized out>) at ProtectedQueue.cc:120
#3  0x0000000000e4c2ad in EThread::DefaultTailHandler::waitForActivity (timeout=60000000, this=<optimized out>) at I_EThread.h:368
#4  EThread::execute_regular (this=0x2b3a35102800) at UnixEThread.cc:272
#5  0x0000000000e47630 in spawn_thread_internal (a=0x60600005d320) at Thread.cc:85
#6  0x00002b3a2514be25 in start_thread (arg=0x2b3a3592a700) at pthread_create.c:308
#7  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 22 (Thread 0x2b3a35713700 (LWP 14835)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000e4697e in ink_cond_timedwait (t=0x2b3a35712ad0, mp=0x2b3a346e5330, cp=0x2b3a346e5358) at ../../lib/ts/ink_thread.h:259
#2  ProtectedQueue::wait (this=0x2b3a346e5310, timeout=<optimized out>) at ProtectedQueue.cc:120
#3  0x0000000000e4c2ad in EThread::DefaultTailHandler::waitForActivity (timeout=60000000, this=<optimized out>) at I_EThread.h:368
#4  EThread::execute_regular (this=0x2b3a345e4800) at UnixEThread.cc:272
#5  0x0000000000e47630 in spawn_thread_internal (a=0x60600005d260) at Thread.cc:85
#6  0x00002b3a2514be25 in start_thread (arg=0x2b3a35713700) at pthread_create.c:308
#7  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 21 (Thread 0x2b3a2ddce700 (LWP 14834)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2ddcdc60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000dd10) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2dbc0800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x606000038a80) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2ddce700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 20 (Thread 0x2b3a2dba8700 (LWP 14833)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2dba7c60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000dca0) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2d99a800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x606000038a20) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2dba8700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 19 (Thread 0x2b3a2d982700 (LWP 14832)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2d981c60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000dc30) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2d774800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x6060000389c0) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2d982700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 18 (Thread 0x2b3a2d75c700 (LWP 14831)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2d75bc60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000dbc0) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2d54e800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x606000038960) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2d75c700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 17 (Thread 0x2b3a2d536700 (LWP 14830)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2d535c60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000db50) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2d328800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x606000038900) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2d536700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 16 (Thread 0x2b3a2d310700 (LWP 14829)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2d30fc60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000dae0) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2d102800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x6060000388a0) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2d310700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 15 (Thread 0x2b3a2d0ea700 (LWP 14828)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2d0e9c60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000da70) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2cedc800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x606000038840) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2d0ea700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 14 (Thread 0x2b3a2cec4700 (LWP 14825)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000bada3d in ink_cond_timedwait (t=0x2b3a2cec3c60, mp=0x60f0000049a0, cp=0x60f0000049c8) at ../../lib/ts/ink_thread.h:259
#2  aio_thread_main (arg=0x60700000da00) at AIO.cc:503
#3  AIOThreadInfo::start (this=<optimized out>, event=<optimized out>, e=<optimized out>) at AIO.cc:196
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2ccb6800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x6060000387e0) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2cec4700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 13 (Thread 0x2b3a2cc9e700 (LWP 14823)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000e4697e in ink_cond_timedwait (t=0x2b3a2cc9dad0, mp=0x2b3a2ca91330, cp=0x2b3a2ca91358) at ../../lib/ts/ink_thread.h:259
#2  ProtectedQueue::wait (this=0x2b3a2ca91310, timeout=<optimized out>) at ProtectedQueue.cc:120
#3  0x0000000000e4c2ad in EThread::DefaultTailHandler::waitForActivity (timeout=60000000, this=<optimized out>) at I_EThread.h:368
#4  EThread::execute_regular (this=0x2b3a2c990800) at UnixEThread.cc:272
#5  0x0000000000e47630 in spawn_thread_internal (a=0x6060000386c0) at Thread.cc:85
#6  0x00002b3a2514be25 in start_thread (arg=0x2b3a2cc9e700) at pthread_create.c:308
#7  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 12 (Thread 0x2b3a2c274700 (LWP 14822)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x00002b3a2358e2a0 in EventNotify::wait (this=0x602000015cb0) at EventNotify.cc:88
#2  0x000000000084ad7a in Log::flush_thread_main () at Log.cc:1373
#3  0x00000000008550bd in LoggingFlushContinuation::mainEvent (this=<optimized out>) at Log.cc:290
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2c066800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x6060000352a0) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2c274700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 11 (Thread 0x2b3a2c04e700 (LWP 14821)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x00002b3a2358e2a0 in EventNotify::wait (this=0x602000015c98) at EventNotify.cc:88
#2  0x000000000084a013 in Log::preproc_thread_main (args=<optimized out>) at Log.cc:1256
#3  0x000000000085509d in LoggingPreprocContinuation::mainEvent (this=<optimized out>) at Log.cc:277
#4  0x0000000000e4cf29 in Continuation::handleEvent (data=<optimized out>, event=1, this=<optimized out>) at I_Continuation.h:160
#5  EThread::execute (this=0x2b3a2be40800) at UnixEThread.cc:317
#6  0x0000000000e47630 in spawn_thread_internal (a=0x6060000351e0) at Thread.cc:85
#7  0x00002b3a2514be25 in start_thread (arg=0x2b3a2c04e700) at pthread_create.c:308
#8  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 10 (Thread 0x2b3a2bc48700 (LWP 14820)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000d28982 in PollCont::do_poll (timeout=60000000, this=<optimized out>) at UnixNet.cc:158
#2  NetHandler::waitForActivity (this=0x2b3a2a6db4c0, timeout=<optimized out>) at UnixNet.cc:477
#3  0x0000000000e4c9d2 in EThread::execute_regular (this=0x2b3a2a6d7800) at UnixEThread.cc:272
#4  0x0000000000e47630 in spawn_thread_internal (a=0x606000016460) at Thread.cc:85
#5  0x00002b3a2514be25 in start_thread (arg=0x2b3a2bc48700) at pthread_create.c:308
#6  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 9 (Thread 0x2b3a2b98e700 (LWP 14819)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000d28982 in PollCont::do_poll (timeout=60000000, this=<optimized out>) at UnixNet.cc:158
#2  NetHandler::waitForActivity (this=0x2b3a2a5cc4c0, timeout=<optimized out>) at UnixNet.cc:477
#3  0x0000000000e4c9d2 in EThread::execute_regular (this=0x2b3a2a5c8800) at UnixEThread.cc:272
#4  0x0000000000e47630 in spawn_thread_internal (a=0x6060000163a0) at Thread.cc:85
#5  0x00002b3a2514be25 in start_thread (arg=0x2b3a2b98e700) at pthread_create.c:308
#6  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 8 (Thread 0x2b3a2b6d4700 (LWP 14818)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000d28982 in PollCont::do_poll (timeout=60000000, this=<optimized out>) at UnixNet.cc:158
#2  NetHandler::waitForActivity (this=0x2b3a2a4bd4c0, timeout=<optimized out>) at UnixNet.cc:477
#3  0x0000000000e4c9d2 in EThread::execute_regular (this=0x2b3a2a4b9800) at UnixEThread.cc:272
#4  0x0000000000e47630 in spawn_thread_internal (a=0x6060000162e0) at Thread.cc:85
#5  0x00002b3a2514be25 in start_thread (arg=0x2b3a2b6d4700) at pthread_create.c:308
#6  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 7 (Thread 0x2b3a2b41a700 (LWP 14817)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000d28982 in PollCont::do_poll (timeout=60000000, this=<optimized out>) at UnixNet.cc:158
#2  NetHandler::waitForActivity (this=0x2b3a2a3ae4c0, timeout=<optimized out>) at UnixNet.cc:477
#3  0x0000000000e4c9d2 in EThread::execute_regular (this=0x2b3a2a3aa800) at UnixEThread.cc:272
#4  0x0000000000e47630 in spawn_thread_internal (a=0x606000016220) at Thread.cc:85
#5  0x00002b3a2514be25 in start_thread (arg=0x2b3a2b41a700) at pthread_create.c:308
#6  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 6 (Thread 0x2b3a2b160700 (LWP 14816)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000d28982 in PollCont::do_poll (timeout=60000000, this=<optimized out>) at UnixNet.cc:158
#2  NetHandler::waitForActivity (this=0x2b3a2a29f4c0, timeout=<optimized out>) at UnixNet.cc:477
#3  0x0000000000e4c9d2 in EThread::execute_regular (this=0x2b3a2a29b800) at UnixEThread.cc:272
#4  0x0000000000e47630 in spawn_thread_internal (a=0x606000016160) at Thread.cc:85
#5  0x00002b3a2514be25 in start_thread (arg=0x2b3a2b160700) at pthread_create.c:308
#6  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 5 (Thread 0x2b3a2aea6700 (LWP 14815)):
#0  0x000000000083fedf in std::vector<PriorityQueueEntry<Http2DependencyTree::Node*>*, std::allocator<PriorityQueueEntry<Http2DependencyTree::Node*>*> >::empty (this=0x60300007cd20) at /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/stl_vector.h:760
#1  PriorityQueue<Http2DependencyTree::Node*, PriorityQueueLess<Http2DependencyTree::Node*> >::empty (this=0x60300007cd20) at ../../lib/ts/PriorityQueue.h:92
#2  PriorityQueue<Http2DependencyTree::Node*, PriorityQueueLess<Http2DependencyTree::Node*> >::_bubble_down (this=0x60300007cd20, index=<optimized out>) at ../../lib/ts/PriorityQueue.h:234
#3  0x000000000082fcb1 in PriorityQueue<Http2DependencyTree::Node*, PriorityQueueLess<Http2DependencyTree::Node*> >::update (increased=true, entry=<optimized out>, this=<optimized out>) at ../../lib/ts/PriorityQueue.h:189
#4  Http2DependencyTree::Tree<Http2Stream*>::update (node=0x60700001fac0, node@entry=0x60700001fa50, sent=sent@entry=0, this=<optimized out>) at Http2DependencyTree.h:422
#5  0x0000000000830d41 in Http2DependencyTree::Tree<Http2Stream*>::deactivate (this=<optimized out>, sent=0, node=<optimized out>) at Http2DependencyTree.h:411
#6  Http2ConnectionState::delete_stream (this=this@entry=0x2b3a36210e30, stream=stream@entry=0x2b3a36233800) at Http2ConnectionState.cc:1192
#7  0x0000000000832976 in Http2ConnectionState::cleanup_streams (this=this@entry=0x2b3a36210e30) at Http2ConnectionState.cc:1158
#8  0x0000000000834d8d in Http2ConnectionState::main_event_handler (this=0x2b3a36210e30, event=<optimized out>, edata=<optimized out>) at Http2ConnectionState.cc:897
#9  0x000000000080f195 in Continuation::handleEvent (data=<optimized out>, event=<optimized out>, this=0x2b3a36210e30) at /usr/local/src/trafficserver/iocore/eventsystem/I_Continuation.h:160
#10 send_connection_event (cont=cont@entry=0x2b3a36210e30, event=event@entry=2252, edata=edata@entry=0x2b3a36210b60) at Http2ClientSession.cc:58
#11 0x000000000081416f in Http2ClientSession::do_io_close (this=0x2b3a36210b60, alerrno=<optimized out>) at Http2ClientSession.cc:261
#12 0x000000000081231a in Http2ClientSession::main_event_handler (this=0x2b3a36210b60, event=<optimized out>, edata=0x63400005ed90) at Http2ClientSession.cc:334
#13 0x0000000000d64796 in Continuation::handleEvent (data=0x63400005ed90, event=<optimized out>, this=<optimized out>) at /usr/local/src/trafficserver/iocore/eventsystem/I_Continuation.h:160
#14 read_signal_and_update (event=<optimized out>, vc=0x63400005ec40) at UnixNetVConnection.cc:83
#15 0x0000000000d67ba7 in UnixNetVConnection::mainEvent (this=0x63400005ec40, event=<optimized out>, e=<optimized out>) at UnixNetVConnection.cc:1189
#16 0x0000000000d36395 in Continuation::handleEvent (data=0x62c000006b20, event=1, this=0x63400005ec40) at /usr/local/src/trafficserver/iocore/eventsystem/I_Continuation.h:160
#17 InactivityCop::check_inactivity (this=<optimized out>, event=<optimized out>, e=<optimized out>) at UnixNet.cc:81
#18 0x0000000000e48bb0 in Continuation::handleEvent (data=0x62c000006b20, event=2, this=<optimized out>) at I_Continuation.h:160
#19 EThread::process_event (this=this@entry=0x2b3a2a18c800, e=e@entry=0x62c000006b20, calling_code=<optimized out>) at UnixEThread.cc:131
#20 0x0000000000e4bec6 in EThread::execute_regular (this=0x2b3a2a18c800) at UnixEThread.cc:244
#21 0x0000000000e47630 in spawn_thread_internal (a=0x6060000160a0) at Thread.cc:85
#22 0x00002b3a2514be25 in start_thread (arg=0x2b3a2aea6700) at pthread_create.c:308
#23 0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 4 (Thread 0x2b3a2abf4700 (LWP 14814)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000d28982 in PollCont::do_poll (timeout=60000000, this=<optimized out>) at UnixNet.cc:158
#2  NetHandler::waitForActivity (this=0x2b3a2a0814c0, timeout=<optimized out>) at UnixNet.cc:477
#3  0x0000000000e4c9d2 in EThread::execute_regular (this=0x2b3a2a07d800) at UnixEThread.cc:272
#4  0x0000000000e47630 in spawn_thread_internal (a=0x606000015fe0) at Thread.cc:85
#5  0x00002b3a2514be25 in start_thread (arg=0x2b3a2abf4700) at pthread_create.c:308
#6  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 3 (Thread 0x2b3a2a9e5700 (LWP 14813)):
#0  0x00002b3a25e82183 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000d28982 in PollCont::do_poll (timeout=60000000, this=<optimized out>) at UnixNet.cc:158
#2  NetHandler::waitForActivity (this=0x2b3a29f724c0, timeout=<optimized out>) at UnixNet.cc:477
#3  0x0000000000e4c9d2 in EThread::execute_regular (this=0x2b3a29f6e800) at UnixEThread.cc:272
#4  0x0000000000e47630 in spawn_thread_internal (a=0x606000015f20) at Thread.cc:85
#5  0x00002b3a2514be25 in start_thread (arg=0x2b3a2a9e5700) at pthread_create.c:308
#6  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 2 (Thread 0x2b3a29a00700 (LWP 14810)):
#0  0x00002b3a25e78c73 in select () at ../sysdeps/unix/syscall-template.S:81
#1  0x00000000009bc234 in mgmt_select (nfds=nfds@entry=1024, readfds=readfds@entry=0x2b3a299ffc30, writefds=writefds@entry=0x0, errorfds=errorfds@entry=0x0, timeout=timeout@entry=0x2b3a299ffb70) at MgmtSocket.cc:195
#2  0x00000000009b1b90 in ProcessManager::pollLMConnection (this=0x60c000002b00) at ProcessManager.cc:373
#3  0x00000000009b276e in ProcessManager::processManagerThread (arg=<optimized out>) at ProcessManager.cc:169
#4  0x00002b3a2514be25 in start_thread (arg=0x2b3a29a00700) at pthread_create.c:308
#5  0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
Thread 1 (Thread 0x2b3a2236d040 (LWP 14809)):
#0  0x00002b3a25e4856d in nanosleep () at ../sysdeps/unix/syscall-template.S:81
#1  0x00002b3a25e48404 in __sleep (seconds=0, seconds@entry=1) at ../sysdeps/unix/sysv/linux/sleep.c:137
#2  0x00000000004b17f3 in main (argv=<optimized out>) at traffic_server/traffic_server.cc:2013


(gdb) thread apply 5 bt full
Thread 5 (Thread 0x2b3a2aea6700 (LWP 14815)):
#0  0x000000000083ff9e in PriorityQueue<Http2DependencyTree::Node*, PriorityQueueLess<Http2DependencyTree::Node*> >::_bubble_down (this=0x60300007ccc0, index=0) at ../../lib/ts/PriorityQueue.h:244
        left = 1
        right = <optimized out>
        smaller = <optimized out>
#1  0x000000000082fcb1 in PriorityQueue<Http2DependencyTree::Node*, PriorityQueueLess<Http2DependencyTree::Node*> >::update (increased=true, entry=<optimized out>, this=<optimized out>) at ../../lib/ts/PriorityQueue.h:189
        increased = true
#2  Http2DependencyTree::Tree<Http2Stream*>::update (node=0x60700001fba0, node@entry=0x60700001fa50, sent=sent@entry=0, this=<optimized out>) at Http2DependencyTree.h:422
No locals.
#3  0x0000000000830d41 in Http2DependencyTree::Tree<Http2Stream*>::deactivate (this=<optimized out>, sent=0, node=<optimized out>) at Http2DependencyTree.h:411
No locals.
#4  Http2ConnectionState::delete_stream (this=this@entry=0x2b3a36210e30, stream=stream@entry=0x2b3a36233800) at Http2ConnectionState.cc:1192
        node = 0x60700001fa50
        __FUNCTION__ = "delete_stream"
#5  0x0000000000832976 in Http2ConnectionState::cleanup_streams (this=this@entry=0x2b3a36210e30) at Http2ConnectionState.cc:1158
        next = 0x2b3a36233c00
        s = 0x2b3a36233800
#6  0x0000000000834d8d in Http2ConnectionState::main_event_handler (this=0x2b3a36210e30, event=<optimized out>, edata=<optimized out>) at Http2ConnectionState.cc:897
        __FUNCTION__ = "main_event_handler"
#7  0x000000000080f195 in Continuation::handleEvent (data=<optimized out>, event=<optimized out>, this=0x2b3a36210e30) at /usr/local/src/trafficserver/iocore/eventsystem/I_Continuation.h:160
No locals.
#8  send_connection_event (cont=cont@entry=0x2b3a36210e30, event=event@entry=2252, edata=edata@entry=0x2b3a36210b60) at Http2ClientSession.cc:58
No locals.
#9  0x000000000081416f in Http2ClientSession::do_io_close (this=0x2b3a36210b60, alerrno=<optimized out>) at Http2ClientSession.cc:261
        __FUNCTION__ = "do_io_close"
#10 0x000000000081231a in Http2ClientSession::main_event_handler (this=0x2b3a36210b60, event=<optimized out>, edata=0x63400005ed90) at Http2ClientSession.cc:334
        retval = <optimized out>
        e = 0x63400005ed90
        __FUNCTION__ = "main_event_handler"
#11 0x0000000000d64796 in Continuation::handleEvent (data=0x63400005ed90, event=<optimized out>, this=<optimized out>) at /usr/local/src/trafficserver/iocore/eventsystem/I_Continuation.h:160
No locals.
#12 read_signal_and_update (event=<optimized out>, vc=0x63400005ec40) at UnixNetVConnection.cc:83
No locals.
#13 0x0000000000d67ba7 in UnixNetVConnection::mainEvent (this=0x63400005ec40, event=<optimized out>, e=<optimized out>) at UnixNetVConnection.cc:1189
        signal_event = 105
        reader_cont = 0x2b3a36210b60
        writer_cont = <optimized out>
        signal_timeout_at = 0x63400005ee98
        t = 0x0
        signal_timeout = <synthetic pointer>
#14 0x0000000000d36395 in Continuation::handleEvent (data=0x62c000006b20, event=1, this=0x63400005ec40) at /usr/local/src/trafficserver/iocore/eventsystem/I_Continuation.h:160
No locals.
#15 InactivityCop::check_inactivity (this=<optimized out>, event=<optimized out>, e=<optimized out>) at UnixNet.cc:81
        vc = 0x63400005ec40
        __FUNCTION__ = "check_inactivity"
#16 0x0000000000e48bb0 in Continuation::handleEvent (data=0x62c000006b20, event=2, this=<optimized out>) at I_Continuation.h:160
No locals.
#17 EThread::process_event (this=this@entry=0x2b3a2a18c800, e=e@entry=0x62c000006b20, calling_code=<optimized out>) at UnixEThread.cc:131
        c_temp = <optimized out>
#18 0x0000000000e4bec6 in EThread::execute_regular (this=0x2b3a2a18c800) at UnixEThread.cc:244
        done_one = true
        sleep_time = <optimized out>
        e = 0x62c000006b20
        next_time = <optimized out>
        delta = <optimized out>
        prev_metric = 0x2b3a2a290f30
        METRIC_INIT = {
          _loop_time = {
            _start = 0,
            _min = 9223372036854775807,
            _max = 0
          },
          _events = {
            _min = 2147483647,
            _max = 0,
            _total = 0
          },
          _count = 0,
          _wait = 0
        }
#19 0x0000000000e47630 in spawn_thread_internal (a=0x6060000160a0) at Thread.cc:85
        p = 0x6060000160a0
#20 0x00002b3a2514be25 in start_thread (arg=0x2b3a2aea6700) at pthread_create.c:308
        __res = <optimized out>
        pd = 0x2b3a2aea6700
        now = <optimized out>
        unwind_buf = {
          cancel_jmp_buf = {{
              jmp_buf = {47528828102400, -6944238515488047313, 0, 2101248, 0, 47528828102400, -3903127703691215057, -3903123386193821905},
              mask_was_saved = 0
            }},
          priv = {
            pad = {0x0, 0x0, 0x0, 0x0},
            data = {
              prev = 0x0,
              cleanup = 0x0,
              canceltype = 0
            }
          }
        }
        not_first_call = <optimized out>
        pagesize_m1 = <optimized out>
        sp = <optimized out>
        freesize = <optimized out>
#21 0x00002b3a25e81bad in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
No locals.

Metadata

Metadata

Assignees

No one assigned

    Labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions