Kernel BUG at net/core/skbuff.c:2169 during transform HTTP/1.1 response into HTTP/2

[b3b]

Scope

Kernel panic occurs when:

• backend returns minimum headers + several KB of data
• H2 request is made

DMESG

[  461.140300] [tempesta fw] Warning: Unable to transform HTTP/1.1 data into HTTP/2 format: free space exhausted (accumulated length: 50
[  461.141858] ------------[ cut here ]------------
[  461.143611] kernel BUG at net/core/skbuff.c:2169!
[  461.144401] invalid opcode: 0000 [#1] SMP NOPTI
[  461.145065] CPU: 1 PID: 20 Comm: ksoftirqd/1 Kdump: loaded Tainted: G           OE     5.10.35+ #1
[  461.146024] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.15.0-1 04/01/2014
[  461.146877] RIP: 0010:skb_put+0x1e/0x50
[  461.147444] Code: 41 bc f4 ff ff ff eb de 0f 1f 40 00 0f 1f 44 00 00 8b 87 b8 00 00 00 48 89 c2 48 03 87 c0 00 00 00 f6 87 83 00 00 00 01 74 02 <0f> 0b 8b 4f 74 85 c9 75 f7 01 f2 01 77 70 89 97 b8 00 00 00 3b 97
[  461.149378] RSP: 0018:ffffad81000b34c8 EFLAGS: 00010206
[  461.150163] RAX: ffff97dae94b96c0 RBX: 0000000000000000 RCX: 0000000000000a12
[  461.151717] RDX: 00000000000001c0 RSI: 0000000000000000 RDI: ffff97dae94b9400
[  461.152644] RBP: ffffad81000b3520 R08: 00000000ffffffe4 R09: ffffad81000b3258
[  461.153492] R10: 0000000000000001 R11: 0000000000000001 R12: 0000000000000001
[  461.154374] R13: 0000000000000000 R14: ffff97dac9dab188 R15: ffffad81000b3580
[  461.155228] FS:  0000000000000000(0000) GS:ffff97dbfbd00000(0000) knlGS:0000000000000000
[  461.156146] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  461.156921] CR2: 00007f817fd26ee8 CR3: 0000000123c08006 CR4: 0000000000770ee0
[  461.157842] PKRU: 55555554
[  461.158479] Call Trace:
[  461.159080]  ? tfw_http_msg_expand_data+0x2a3/0x380 [tempesta_fw]
[  461.159912]  ? tfw_h2_msg_rewrite_data+0x15/0x20 [tempesta_fw]
[  461.160709]  tfw_hpack_hdr_expand+0x78/0x170 [tempesta_fw]
[  461.161518]  ? tfw_hpack_hdr_add+0xde/0x130 [tempesta_fw]
[  461.162359]  tfw_hpack_encode+0x662/0x990 [tempesta_fw]
[  461.163182]  tfw_h2_add_hdr_date+0xbb/0x100 [tempesta_fw]
[  461.163993]  tfw_http_resp_cache_cb+0x4c6/0x600 [tempesta_fw]
[  461.164760]  ? __dynamic_pr_debug+0x8a/0xb0
[  461.165436]  ? tfw_http_conn_drop+0x230/0x230 [tempesta_fw]
[  461.166219]  tfw_cache_do_action+0x491/0xf10 [tempesta_fw]
[  461.167487]  ? vprintk_default+0x1d/0x20
[  461.168139]  ? vprintk_func+0x67/0x100
[  461.168784]  ? tfw_http_conn_drop+0x230/0x230 [tempesta_fw]
[  461.169570]  ? tfw_hash_str_len+0xbd/0x170 [tempesta_fw]
[  461.170379]  ? tfw_http_conn_drop+0x230/0x230 [tempesta_fw]
[  461.171189]  ? tfw_http_conn_drop+0x230/0x230 [tempesta_fw]
[  461.172003]  tfw_cache_process+0xbb/0x2a0 [tempesta_fw]
[  461.172706]  ? tfw_cache_process+0xbb/0x2a0 [tempesta_fw]
[  461.173416]  tfw_http_resp_cache+0xf7/0x1a0 [tempesta_fw]
[  461.174166]  tfw_http_msg_process_generic+0x4f9/0x670 [tempesta_fw]
[  461.174920]  tfw_http_msg_process+0x38/0x50 [tempesta_fw]
[  461.175628]  tfw_connection_recv+0x59/0xa0 [tempesta_fw]
[  461.176323]  ss_tcp_process_data+0x1e7/0x3f0 [tempesta_fw]
[  461.177029]  ss_tcp_data_ready+0x44/0xb0 [tempesta_fw]
[  461.177706]  tcp_data_ready+0x2b/0xd0
[  461.178314]  tcp_rcv_established+0x5a7/0x670
[  461.178935]  tcp_v4_do_rcv+0x140/0x200
[  461.179512]  tcp_v4_rcv+0xcfd/0xe10
[  461.180063]  ip_protocol_deliver_rcu+0x30/0x1b0
[  461.180672]  ip_local_deliver_finish+0x48/0x60
[  461.181274]  ip_local_deliver+0x72/0x110
[  461.181843]  ? tcp_v4_early_demux+0xfa/0x160
[  461.182496]  ? ip_rcv_finish_core.isra.0+0x146/0x420
[  461.183233]  ip_rcv_finish+0x87/0xa0
[  461.183765]  ip_rcv+0xcc/0xe0
[  461.184252]  ? ip_rcv_finish_core.isra.0+0x420/0x420
[  461.184848]  __netif_receive_skb_one_core+0x88/0xa0
[  461.185432]  __netif_receive_skb+0x18/0x60
[  461.185987]  process_backlog+0xa9/0x160
[  461.186569]  net_rx_action+0x13e/0x390
[  461.187132]  ? rcu_core+0x3ff/0x500
[  461.187715]  __do_softirq+0xd9/0x291
[  461.188263]  run_ksoftirqd+0x2b/0x40
[  461.188799]  smpboot_thread_fn+0xd0/0x170
[  461.189359]  kthread+0x114/0x150
[  461.189870]  ? sort_range+0x30/0x30
[  461.190386]  ? kthread_park+0x90/0x90
[  461.190915]  ret_from_fork+0x1f/0x30
[  461.191432] Modules linked in: tempesta_fw(OE) tempesta_db(OE) tempesta_tls(OE) tempesta_lib(OE) sha256_ssse3 sha512_ssse3 nls_utf8 isofs nls_iso8859_1 dm_multipath scsi_dh_rdac scsi_dh_emc scsi_dh_alua intel_rapl_msr intel_rapl_common kvm_intel kvm ppdev input_leds joydev parport_pc parport serio_raw mac_hid qemu_fw_cfg sch_fq_codel ipmi_devintf ipmi_msghandler msr ip_tables x_tables autofs4 btrfs blake2b_generic zstd_compress raid10 raid456 async_raid6_recov async_memcpy async_pq async_xor async_tx xor raid6_pq libcrc32c raid1 raid0 multipath linear bochs_drm drm_vram_helper drm_ttm_helper crct10dif_pclmul ttm crc32_pclmul drm_kms_helper ghash_clmulni_intel syscopyarea sysfillrect sysimgblt fb_sys_fops cec drm aesni_intel crypto_simd i2c_piix4 cryptd glue_helper psmouse virtio_net net_failover failover pata_acpi floppy [last unloaded: tempesta_lib]

Related issues: maybe duplicate of #1669

Testing

Test to reproduce: http2_general.test_h2_headers.MissingDateServerWithLargeBodyTest

[RomanBelozerov]

I don't have kernel panic for this test, but Tempesta crashes.

DMESG

[ 4916.939986] ------------[ cut here ]------------
[ 4916.940983] WARNING: CPU: 1 PID: 10685 at /root/tempesta/fw/http_msg.c:1398 tfw_h2_msg_rewrite_data_common+0xdd/0x2c0 [tempesta_fw]
[ 4916.943021] Modules linked in: tempesta_fw(OE) tempesta_db(OE) tempesta_tls(OE) tempesta_lib(OE) sha256_ssse3 sha512_ssse3 rfcomm bnep vsock_loopback vmw_vsock_virtio_transport_common vmw_vsock_vmci_transport vsock nls_iso8859_1 binfmt_misc intel_rapl_msr intel_rapl_common crct10dif_pclmul ghash_clmulni_intel aesni_intel crypto_simd cryptd glue_helper snd_ens1371 snd_ac97_codec gameport ac97_bus btusb snd_pcm btrtl btbcm btintel snd_seq_midi bluetooth snd_seq_midi_event snd_rawmidi snd_seq snd_seq_device vmw_balloon snd_timer joydev snd input_leds serio_raw ecdh_generic ecc vmw_vmci soundcore mac_hid sch_fq_codel vmwgfx ttm drm_kms_helper cec rc_core fb_sys_fops syscopyarea sysfillrect sysimgblt ipmi_devintf ipmi_msghandler drm msr parport_pc ppdev lp parport ramoops reed_solomon pstore_blk efi_pstore pstore_zone ip_tables x_tables autofs4 hid_generic usbhid hid psmouse mptspi mptscsih crc32_pclmul mptbase e1000 pata_acpi ahci libahci scsi_transport_spi i2c_piix4
[ 4916.943065]  [last unloaded: tempesta_lib]
[ 4916.958469] CPU: 1 PID: 10685 Comm: python3 Kdump: loaded Tainted: G        W  OE     5.10.35+ #1
[ 4916.959977] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
[ 4916.961892] RIP: 0010:tfw_h2_msg_rewrite_data_common+0xdd/0x2c0 [tempesta_fw]
[ 4916.963124] Code: 8b 78 08 48 c1 fa 06 8b 40 0c 48 c1 e2 0c 48 03 15 b8 48 c7 c4 48 01 c2 49 8b 7e 48 44 89 fe 48 89 f8 48 29 d0 48 39 f0 72 23 <0f> 0b b8 ea ff ff ff 48 83 c4 38 5b 41 5c 41 5d 41 5e 41 5f 5d c3
[ 4916.966312] RSP: 0018:ffffb1838003c610 EFLAGS: 00010216
[ 4916.967169] RAX: 0000000000545019 RBX: 0000000000000001 RCX: ffffffffc06c80c0
[ 4916.968300] RDX: ffff9860cdf73000 RSI: 000000000000001f RDI: ffff9860ce4b8019
[ 4916.969576] RBP: ffffb1838003c670 R08: 0000000000000012 R09: 0000000000000076
[ 4916.971519] R10: 0000000000001000 R11: 0000000000000012 R12: 0000000000000000
[ 4916.978907] R13: ffffb1838003c6b0 R14: ffff9860ce230130 R15: 000000000000001f
[ 4916.983327] FS:  00007f64605bb700(0000) GS:ffff9862f5e40000(0000) knlGS:0000000000000000
[ 4916.986629] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 4916.988671] CR2: 00007f6460aabdb0 CR3: 000000004a2fe006 CR4: 0000000000370ee0
[ 4916.990394] Call Trace:
[ 4916.990963]  <IRQ>
[ 4916.991427]  ? __hash_calc+0xc0/0xc0 [tempesta_lib]
[ 4916.992256]  tfw_h2_msg_rewrite_data+0x15/0x20 [tempesta_fw]
[ 4916.993415]  tfw_hpack_hdr_add+0x5e/0x130 [tempesta_fw]
[ 4916.994370]  tfw_hpack_encode+0x7d7/0x990 [tempesta_fw]
[ 4916.995251]  tfw_h2_msg_hdr_add.constprop.0+0x92/0xb0 [tempesta_fw]
[ 4916.996358]  tfw_http_resp_cache_cb+0x4d6/0x600 [tempesta_fw]
[ 4916.997503]  ? __dev_queue_xmit+0x3b3/0x910
[ 4916.998254]  ? tfw_http_conn_drop+0x230/0x230 [tempesta_fw]
[ 4916.999168]  tfw_cache_do_action+0x491/0xf10 [tempesta_fw]
[ 4917.000102]  ? dev_queue_xmit+0x10/0x20
[ 4917.000763]  ? ip_finish_output2+0x1ab/0x590
[ 4917.001630]  ? tfw_http_conn_drop+0x230/0x230 [tempesta_fw]
[ 4917.002568]  ? tfw_hash_str_len+0xbd/0x170 [tempesta_fw]
[ 4917.003471]  ? tfw_http_conn_drop+0x230/0x230 [tempesta_fw]
[ 4917.004422]  ? tfw_http_conn_drop+0x230/0x230 [tempesta_fw]
[ 4917.005489]  tfw_cache_process+0xbb/0x2a0 [tempesta_fw]
[ 4917.006370]  ? tfw_cache_process+0xbb/0x2a0 [tempesta_fw]
[ 4917.007271]  ? tfw_gfsm_move+0x14e/0x200 [tempesta_fw]
[ 4917.008113]  tfw_http_resp_cache+0xf7/0x1a0 [tempesta_fw]
[ 4917.009159]  tfw_http_msg_process_generic+0x4f9/0x670 [tempesta_fw]
[ 4917.010190]  ? pg_skb_alloc+0x43b/0x480
[ 4917.010816]  ? __alloc_skb+0x44/0x200
[ 4917.011420]  tfw_http_msg_process+0x38/0x50 [tempesta_fw]
[ 4917.012322]  tfw_connection_recv+0x59/0xa0 [tempesta_fw]
[ 4917.013309]  ss_tcp_process_data+0x1e7/0x3f0 [tempesta_fw]
[ 4917.014222]  ss_tcp_data_ready+0x44/0xb0 [tempesta_fw]
[ 4917.015059]  tcp_data_ready+0x2b/0xd0
[ 4917.015659]  tcp_rcv_established+0x5a7/0x670
[ 4917.016367]  tcp_v4_do_rcv+0x140/0x200
[ 4917.017101]  tcp_v4_rcv+0xcfd/0xe10
[ 4917.017734]  ip_protocol_deliver_rcu+0x30/0x1b0
[ 4917.018480]  ip_local_deliver_finish+0x48/0x60
[ 4917.019206]  ip_local_deliver+0x72/0x110
[ 4917.019858]  ? tfw_ipv4_nf_hook+0xb9/0x130 [tempesta_fw]
[ 4917.020728]  ip_rcv_finish+0x87/0xa0
[ 4917.021429]  ip_rcv+0xcc/0xe0
[ 4917.021941]  ? ip_rcv_finish_core.isra.0+0x420/0x420
[ 4917.022747]  __netif_receive_skb_one_core+0x88/0xa0
[ 4917.023555]  __netif_receive_skb+0x18/0x60
[ 4917.024270]  process_backlog+0xa9/0x160
[ 4917.025081]  net_rx_action+0x13e/0x390
[ 4917.025717]  __do_softirq+0xd9/0x291
[ 4917.026287]  asm_call_irq_on_stack+0x12/0x20
[ 4917.026977]  </IRQ>
[ 4917.027323]  do_softirq_own_stack+0x3d/0x50
[ 4917.028017]  do_softirq.part.0+0x46/0x50
[ 4917.028752]  __local_bh_enable_ip+0x50/0x60
[ 4917.029464]  ip_finish_output2+0x1ab/0x590
[ 4917.030126]  ? __cgroup_bpf_run_filter_skb+0x3c3/0x3d0
[ 4917.031213]  __ip_finish_output+0xd8/0x220
[ 4917.032607]  ip_finish_output+0x2d/0xb0
[ 4917.034076]  ip_output+0x7a/0x100
[ 4917.034618]  ip_local_out+0x3d/0x50
[ 4917.035187]  __ip_queue_xmit+0x17a/0x470
[ 4917.035816]  ? get_page_from_freelist+0xed8/0x1100
[ 4917.036571]  ip_queue_xmit+0x15/0x20
[ 4917.037256]  __tcp_transmit_skb+0xa94/0xc90
[ 4917.037976]  tcp_write_xmit+0x453/0x1350
[ 4917.038620]  __tcp_push_pending_frames+0x37/0x100
[ 4917.039382]  tcp_push+0xfc/0x100
[ 4917.039910]  tcp_sendmsg_locked+0xd36/0xe70
[ 4917.040591]  tcp_sendmsg+0x2d/0x50
[ 4917.041254]  inet_sendmsg+0x43/0x70
[ 4917.041830]  sock_sendmsg+0x5e/0x70
[ 4917.042403]  __sys_sendto+0x113/0x190
[ 4917.043009]  ? exit_to_user_mode_prepare+0x3d/0x170
[ 4917.043795]  ? do_user_addr_fault+0x1f2/0x3c0
[ 4917.044520]  __x64_sys_sendto+0x29/0x30
[ 4917.045266]  do_syscall_64+0x38/0x90
[ 4917.045860]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
[ 4917.046678] RIP: 0033:0x7f64641f457c
[ 4917.047265] Code: 89 4c 24 1c e8 85 40 f7 ff 44 8b 54 24 1c 8b 3c 24 45 31 c9 89 c5 48 8b 54 24 10 48 8b 74 24 08 45 31 c0 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 30 89 ef 48 89 04 24 e8 b1 40 f7 ff 48 8b 04
[ 4917.050419] RSP: 002b:00007f64605b9c80 EFLAGS: 00000246 ORIG_RAX: 000000000000002c
[ 4917.051643] RAX: ffffffffffffffda RBX: 00007f64605b9d40 RCX: 00007f64641f457c
[ 4917.052788] RDX: 0000000000000029 RSI: 00007f646225da00 RDI: 0000000000000024
[ 4917.054071] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000
[ 4917.055241] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000
[ 4917.056406] R13: 0000000000000000 R14: 00007f64605b9d40 R15: 00000000006256d0
[ 4917.057683] ---[ end trace 206a85bdfedded09 ]---