Http3 transaction crash

[brbzull0]

After merging from master, I've got the following crash. This seem it was introduced by #9729

trafficserver/proxy/http3/Http3Transaction.cc

Line 309 in eb09664

ink_assert(this->_write_ready_event == e);

(gdb) bt
#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
#1  0x00007ffff7127859 in __GI_abort () at abort.c:79
#2  0x00007ffff7ef0ff1 in ink_abort (message_format=0x7ffff7f5e020 "%s:%d: failed assertion `%s`") at ink_error.cc:99
#3  0x00007ffff7eebe9d in _ink_assert (expression=0x55555609c1b0 "this->_write_ready_event == e", file=0x55555609c158 "Http3Transaction.cc", line=309)
    at ink_assert.cc:37
#4  0x0000555555f7ea02 in HQTransaction::_close_write_ready_event (this=0x7fffd0101f60, e=0x0) at Http3Transaction.cc:309
#5  0x0000555555f80cff in Http3Transaction::state_stream_closed (this=0x7fffd0101f60, event=101, data=0x0) at Http3Transaction.cc:507
#6  0x0000555555a45497 in Continuation::handleEvent (this=0x7fffd0101f60, event=101, data=0x0)
    at /home/dmeden/code/git/trafficserver-h3/iocore/eventsystem/I_Continuation.h:228
#7  0x0000555555f76e3c in Http3App::_handle_bidi_stream_on_write_ready (this=0x7fffd00e3970, event=101, vio=0x7fffd00fef08) at Http3App.cc:338
#8  0x0000555555f75d1c in Http3App::main_event_handler (this=0x7fffd00e3970, event=101, data=0x5555565d6800) at Http3App.cc:141
#9  0x0000555555a45497 in Continuation::handleEvent (this=0x7fffd00e3970, event=101, data=0x5555565d6800)
    at /home/dmeden/code/git/trafficserver-h3/iocore/eventsystem/I_Continuation.h:228
#10 0x0000555555fbbaed in EThread::process_event (this=0x555556b484b0, e=0x5555565d6800, calling_code=101) at UnixEThread.cc:149
#11 0x0000555555fbbd3a in EThread::process_queue (this=0x555556b484b0, NegativeQueue=0x7fffd7efd0f0, ev_count=0x7fffd7efd0a4, nq_count=0x7fffd7efd0a0)
    at UnixEThread.cc:184
#12 0x0000555555fbc18d in EThread::execute_regular (this=0x555556b484b0) at UnixEThread.cc:262
#13 0x0000555555fbc474 in EThread::execute (this=0x555556b484b0) at UnixEThread.cc:334
#14 0x0000555555fbabbc in spawn_thread_internal (a=0x5555565ccc40) at Thread.cc:79
#15 0x00007ffff7780609 in start_thread (arg=<optimized out>) at pthread_create.c:477
#16 0x00007ffff7224133 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
(gdb) f 4
#4  0x0000555555f7ea02 in HQTransaction::_close_write_ready_event (this=0x7fffd0101f60, e=0x0) at Http3Transaction.cc:309
309       ink_assert(this->_write_ready_event == e);
(gdb) print e
$17 = (Event *) 0x0
(gdb) f 8
#8  0x0000555555f75d1c in Http3App::main_event_handler (this=0x7fffd00e3970, event=101, data=0x5555565d6800) at Http3App.cc:141
141           this->_handle_bidi_stream_on_write_ready(event, vio);
(gdb) print data->_location
$18 = (const void *) 0x555555fa62fa <QUICStreamVCAdapter::encourge_write()+232>
(gdb) print data
$19 = (Event *) 0x5555565d6800
(gdb)

the only way to get this was using h2load and using some concurrent requests.

[bryancall]

I am seeing a similar core on docs:

/var/lib/apport/coredump$ sudo gdb /opt/ats/bin/traffic_server core._opt_ats_bin_traffic_server.65534.b48db689-2ba1-449e-a712-0800c7998be0.720880.1253043068
(gdb) bt full
#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
        set = {__val = {0 <repeats 16 times>}}
        pid = <optimized out>
        tid = <optimized out>
        ret = <optimized out>
#1  0x00007f450d24c859 in __GI_abort () at abort.c:79
        save_stage = 1
        act = {__sigaction_handler = {sa_handler = 0x0, sa_sigaction = 0x0}, sa_mask = {__val = {0 <repeats 16 times>}}, sa_flags = 0, sa_restorer = 0x7f4483e0a0b0}
        sigs = {__val = {32, 0 <repeats 15 times>}}
#2  0x00007f450e009ff1 in ink_abort (message_format=0x7f450e077040 "%s:%d: failed assertion `%s`") at ink_error.cc:99
        ap = {{gp_offset = 32, fp_offset = 48, overflow_arg_area = 0x7f4509baed40, reg_save_area = 0x7f4509baec80}}
#3  0x00007f450e004e9d in _ink_assert (expression=0x55df52c6114c "this->_read_ready_event == e", file=0x55df52c61138 "Http3Transaction.cc", line=255) at ink_assert.cc:37
No locals.
#4  0x000055df52b4489c in HQTransaction::_close_read_ready_event (this=0x7f4483e0a090, e=0x0) at Http3Transaction.cc:255
No locals.
#5  0x000055df52b45d09 in Http3Transaction::state_stream_open (this=0x7f4483e0a090, event=100, edata=0x0) at Http3Transaction.cc:441
        lock = {m = {m_ptr = 0x7f4480e00820}, locked_p = true}
        __FUNCTION__ = "state_stream_open"
#6  0x000055df5260d477 in Continuation::handleEvent (this=0x7f4483e0a090, event=100, data=0x0) at /home/maskit/src/trafficserver/iocore/eventsystem/I_Continuation.h:228
No locals.
#7  0x000055df52b3c9b4 in Http3App::_handle_bidi_stream_on_read_ready (this=0x7f44f40d4180, event=100, vio=0x7f4484c07270) at Http3App.cc:248
        lock = {m = {m_ptr = 0x7f4480e00820}, locked_p = true}
        adapter = 0x7f4484c07210
        stream_id = 96
        txn = 0x7f4483e0a090
        __FUNCTION__ = "_handle_bidi_stream_on_read_ready"
#8  0x000055df52b3bd90 in Http3App::main_event_handler (this=0x7f44f40d4180, event=100, data=0x7f4484c074a0) at Http3App.cc:134
        __FUNCTION__ = "main_event_handler"
        vio = 0x7f4484c07270
        adapter = 0x7f4484c07210
        is_bidirectional = true
#9  0x000055df5260d477 in Continuation::handleEvent (this=0x7f44f40d4180, event=100, data=0x7f4484c074a0)
    at /home/maskit/src/trafficserver/iocore/eventsystem/I_Continuation.h:228
No locals.
#10 0x000055df52b81b9b in EThread::process_event (this=0x55df542a9870, e=0x7f4484c074a0, calling_code=100) at UnixEThread.cc:149
        c_temp = 0x7f44f40d4180
        lock = {m = {m_ptr = 0x7f44f40cc2d0}, lock_acquired = true}
        __FUNCTION__ = "process_event"
#11 0x000055df52b81de8 in EThread::process_queue (this=0x55df542a9870, NegativeQueue=0x7f4509baf0f0, ev_count=0x7f4509baf0a4, nq_count=0x7f4509baf0a0) at UnixEThread.cc:184
        e = 0x7f4484c074a0