Crash due to global variable corruption #6946
Description
We are still seeing global variable corruption in prod and digging deeper it looks like it may be somehow related to UserArg table corruption.
The crash pointed to a corrupted http_global_hooks global and upon examining other globals around it, turns out all of those are corrupted as well all the way up to UserArgTable. I suspect there may have been an array boundary underflow as I see garbage entries (see below).
We are still using the old TxnArg API in all our plugins, but, since the CPPAPI (part of ATS Core) has been modified to use the new UserArg API, we've a scenario where we are partly using the new API and old API. Talking to @zwoop , this is designed to still work in 9.0. so, will add some boundary check asserts into UserArgTable writes to determine which of our plugins is corrupting them.
Program terminated with signal 11, Segmentation fault.
#0 head (this=0x2541302541302589) at traffic_server/InkAPI.cc:1339
1339 traffic_server/InkAPI.cc: No such file or directory.
(gdb) bt
#0 head (this=0x2541302541302589) at traffic_server/InkAPI.cc:1339
#1 init (id=TS_HTTP_READ_RESPONSE_HDR_HOOK, feature_hooks=<optimized out>, this=0x2ae24e773ae8) at traffic_server/InkAPI.cc:1427
#2 HttpHookState::init (this=0x2ae24e773ae0, id=TS_HTTP_READ_RESPONSE_HDR_HOOK, global=<optimized out>, ssn=0x2ae20d377600, txn=0x2ae24e773b50) at traffic_server/InkAPI.cc:1370
#3 0x000000000054f3cd in HttpSM::do_api_callout_internal (this=0x2ae24e771920) at HttpSM.cc:5332
#4 0x000000000055d605 in HttpSM::do_api_callout (this=this@entry=0x2ae24e771920) at HttpSM.cc:365
#5 0x000000000054fcab in HttpSM::state_read_server_response_header (this=0x2ae24e771920, event=100, data=0x2ae2ae30c2c0) at HttpSM.cc:2006
#6 0x0000000000551f68 in HttpSM::main_handler (this=0x2ae24e771920, event=100, data=0x2ae2ae30c2c0) at HttpSM.cc:2710
#7 0x0000000000776a03 in handleEvent (data=0x2ae2ae30c2c0, event=100, this=0x2ae24e771920) at /home/svinukon/Traffic/ATS/ats9/ats-core_trunk/ats9/src/iocore/eventsystem/I_Continuation.h:190
#8 read_signal_and_update (event=100, vc=0x2ae2ae30c0e0) at UnixNetVConnection.cc:83
#9 0x000000000077d1ee in read_from_net (nh=0x2ae18a009dd0, vc=0x2ae2ae30c0e0, thread=0x2ae18a006000) at UnixNetVConnection.cc:314
#10 0x0000000000762ee8 in NetHandler::process_ready_list (this=this@entry=0x2ae18a009dd0) at UnixNet.cc:412
#11 0x00000000007631dd in NetHandler::waitForActivity (this=0x2ae18a009dd0, timeout=<optimized out>) at UnixNet.cc:547
#12 0x00000000007c6eba in EThread::execute_regular (this=this@entry=0x2ae18a006000) at UnixEThread.cc:266
#13 0x00000000007c7182 in EThread::execute (this=0x2ae18a006000) at UnixEThread.cc:327
#14 0x00000000007c5529 in spawn_thread_internal (a=0x2ae188569e40) at Thread.cc:92
#15 0x00002ae18652add5 in start_thread () from /lib64/libpthread.so.0
#16 0x00002ae1872dbead in clone () from /lib64/libc.so.6
(gdb) p hostdb_max_iobuf_index
$1 = 808592688
(gdb) p sizeof9hostdb_max_iobuf_index)
p sizeof(No symbol "sizeof9hostdb_max_iobuf_index" in current context.
(gdb) p sizeof(hostdb_max_iobuf_index)
$2 = 4
(gdb) p (int)0x2541302541302589
$3 = 1093674377
(gdb) p http_global_hooks
$4 = (HttpAPIHooks *) 0x2541302541302541
(gdb) p ssl_hooks
$5 = (SslAPIHooks *) 0x3025413025413025
(gdb) p lifecycle_hoooks
No symbol "lifecycle_hoooks" in current context.
(gdb) p lifecycle_hooks
$6 = (LifecycleAPIHooks *) 0x4130254130254130
(gdb) p TS_HTTP_LEN_PUSH
$7 = 807747888
(gdb) p TS_HTTP_LEN_CONNECT
$8 = 807747888
(gdb) p TS_HTTP_METHOD_CONNECT
$9 = 0x2541302541302541 <Address 0x2541302541302541 out of bounds>
(gdb) p TS_HTTP_VALUE_BYTES
$10 = 0x3025413025413025 <Address 0x3025413025413025 out of bounds>
(gdb) p TS_MIME_LEN_ACCEPT
$11 = 1093674305
(gdb) p MIME_LEN_ACCEPT
$12 = 6
(gdb) p TS_MIME_FIELD_ACCEPT
$13 = 0x4130254130254130 <Address 0x4130254130254130 out of bounds>
(gdb) p TS_URL_SCHEME_FILE
$14 = 0x4130254130254130 <Address 0x4130254130254130 out of bounds>
(gdb) p TS_USER_ARGS_COUNT
$15 = TS_USER_ARGS_COUNT
(gdb) p TS_USER_ARGS_GLB
$16 = TS_USER_ARGS_GLB
(gdb) p UserArgTable
$17 = {{{type = 1818585446, name = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x676e6e6576303425 <Address 0x676e6e6576303425 out of bounds>}, _M_string_length = 7885630523655417714, {
_M_local_buf = "%0A%0A%23contrac", _M_allocated_capacity = 3613365951073955877}}, description = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
_M_p = 0x6833322530322574 <Address 0x6833322530322574 out of bounds>}, _M_string_length = 8241978093345726821, {_M_local_buf = "e%20%23%20dynami", _M_allocated_capacity = 2680541338519348581}}}, {type = 841315171, name = {static npos = 18446744073709551615,
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x6c657665646d7263 <Address 0x6c657665646d7263 out of bounds>}, _M_string_length = 2756537384318627951, {_M_local_buf = "includeFollowed=", _M_allocated_capacity = 5072571010795138665}},
description = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x6572702665757274 <Address 0x6572702665757274 out of bounds>}, _M_string_length = 6365935209299863910, {_M_local_buf = "XXXXXXXX&q=hasht",
_M_allocated_capacity = 6365935209750747224}}}, {type = 7563105, name = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xab0868 <UserArgTable+168> ""}, _M_string_length = 0, {
_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}, description = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xab0888 <UserArgTable+200> ""}, _M_string_length = 0, {
_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}}, {type = TS_USER_ARGS_TXN, name = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xab08b0 <UserArgTable+240> ""},
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}, description = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xab08d0 <UserArgTable+272> ""},
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}}, {type = TS_USER_ARGS_TXN, name = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
_M_p = 0xab08f8 <UserArgTable+312> ""}, _M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}, description = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
_M_p = 0xab0918 <UserArgTable+344> ""}, _M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}}, {type = TS_USER_ARGS_TXN, name = {static npos = 18446744073709551615,
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xab0940 <UserArgTable+384> ""}, _M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}, description = {static npos = 18446744073709551615,
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xab0960 <UserArgTable+416> ""}, _M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}}, {type = TS_USER_ARGS_TXN, name = {
static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xab0988 <UserArgTable+456> ""}, _M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}, description = {
static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xab09a8 <UserArgTable+488> ""}, _M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}}, {type = TS_USER_ARGS_TXN,
name = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0xab09d0 <UserArgTable+528> ""}, _M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}, description = {