kernel panic when cache is enabled and server responds with tcp segmentation

[RomanBelozerov]

Scope

Kernel panic occurs when

• response has body;
• TCP chunk of server response < 100 bytes;
• cache is enabled;

See #1408

[  721.041638] ------------[ cut here ]------------
[  721.042549] kernel BUG at /root/tempesta/db/core/htrie.c:636!
[  721.043586] invalid opcode: 0000 [#1] SMP NOPTI
[  721.044397] CPU: 2 PID: 9437 Comm: python3 Tainted: G           OE     5.10.35+ #16
[  721.045735] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
[  721.047711] RIP: 0010:tdb_htrie_extend_rec+0x7a/0x90 [tempesta_db]
[  721.048806] Code: 75 1e 8b 4b 0c 85 c9 7e 21 48 c1 ea 07 f0 0f b1 53 08 85 c0 75 e5 48 83 c4 08 4c 89 c0 5b 5d c3 c1 e0 07 48 8d 5c 05 00 eb d1 <0f> 0b 48 83 c4 08 45 31 c0 4c 89 c0 5b 5d c3 0f 0b 0f 1f 44 00 00
[  721.052086] RSP: 0018:ffffa512404ec788 EFLAGS: 00010246
[  721.053009] RAX: 0000000000000000 RBX: ffff96218f301200 RCX: 0000000000000000
[  721.054254] RDX: 0000000000101280 RSI: 00000000fffffe01 RDI: ffffffffc0c5fb02
[  721.055499] RBP: ffff96218f200000 R08: ffff96218f301280 R09: 0000000000000000
[  721.056739] R10: ffffdc6180000000 R11: ffffa512404ec918 R12: ffffa512404ec890
[  721.057986] R13: 0000000000000000 R14: ffffffffc0d25840 R15: ffffa512404ec938
[  721.059245] FS:  00007fa92a542700(0000) GS:ffff9622da080000(0000) knlGS:0000000000000000
[  721.060676] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  721.061688] CR2: 00007fa92ea71bb0 CR3: 00000001826d6006 CR4: 0000000000370ee0
[  721.063139] Call Trace:
[  721.063635]  <IRQ>
[  721.064387]  __tfw_cache_strcpy+0x7a/0x110 [tempesta_fw]
[  721.066203]  tfw_cache_copy_resp.isra.0+0x122e/0x1ab0 [tempesta_fw]
[  721.067616]  __cache_add_node+0xd1/0x100 [tempesta_fw]
[  721.068576]  tfw_cache_do_action+0x938/0x1020 [tempesta_fw]
[  721.069623]  ? prb_read_valid+0x17/0x20
[  721.070293]  ? tfw_h2_resp_adjust_fwd+0x610/0x610 [tempesta_fw]
[  721.071341]  ? tfw_cache_process+0xb7/0x290 [tempesta_fw]
[  721.072306]  tfw_cache_process+0xb7/0x290 [tempesta_fw]
[  721.073233]  ? tfw_gfsm_move+0x1a5/0x210 [tempesta_fw]
[  721.074155]  tfw_http_resp_cache+0xfb/0x1b0 [tempesta_fw]
[  721.075140]  tfw_http_msg_process_generic+0x4bd/0x5e0 [tempesta_fw]
[  721.076249]  ? __coalesce_frag+0x99/0x150 [tempesta_fw]
[  721.077183]  ? ss_skb_queue_coalesce_tail+0xf5/0x130 [tempesta_fw]
[  721.078328]  tfw_connection_recv+0x52/0xa0 [tempesta_fw]
[  721.079300]  ss_tcp_process_data+0x1f1/0x400 [tempesta_fw]
[  721.080279]  ss_tcp_data_ready+0x3f/0xc0 [tempesta_fw]
[  721.081186]  tcp_rcv_established+0x5d8/0x680
[  721.081939]  tcp_v4_do_rcv+0x131/0x1f0
[  721.082634]  tcp_v4_rcv+0xc2f/0xd80
[  721.083260]  ip_protocol_deliver_rcu+0x2b/0x1b0
[  721.084061]  ip_local_deliver_finish+0x44/0x50
[  721.084841]  __netif_receive_skb_one_core+0x87/0xa0
[  721.085705]  process_backlog+0x96/0x160
[  721.086398]  net_rx_action+0x145/0x3e0
[  721.087070]  ? try_to_wake_up+0x1d1/0x4f0
[  721.087776]  __do_softirq+0xcf/0x284
[  721.088419]  asm_call_irq_on_stack+0x12/0x20
[  721.089208]  </IRQ>
[  721.089582]  do_softirq_own_stack+0x37/0x40
[  721.090306]  do_softirq+0x5e/0x70
[  721.090879]  __local_bh_enable_ip+0x4b/0x50
[  721.091595]  ip_finish_output2+0x1ab/0x590
[  721.092322]  __ip_queue_xmit+0x180/0x410
[  721.093003]  __tcp_transmit_skb+0xa0e/0xbc0
[  721.093744]  tcp_write_xmit+0x401/0x1280
[  721.094447]  __tcp_push_pending_frames+0x32/0xf0
[  721.095268]  tcp_sendmsg_locked+0xa32/0xb50
[  721.095991]  tcp_sendmsg+0x28/0x40
[  721.096580]  sock_sendmsg+0x57/0x60
[  721.097221]  __sys_sendto+0xee/0x150
[  721.097839]  ? do_user_addr_fault+0x1cf/0x3f0
[  721.098591]  ? switch_fpu_return+0x40/0xb0
[  721.099318]  __x64_sys_sendto+0x25/0x30
[  721.099981]  do_syscall_64+0x33/0x80
[  721.100597]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
[  721.101504] RIP: 0033:0x7fa92f0f5494
[  721.102153] Code: 89 4c 24 1c e8 8d f8 ff ff 44 8b 54 24 1c 8b 3c 24 45 31 c9 89 c5 48 8b 54 24 10 48 8b 74 24 08 45 31 c0 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 34 89 ef 48 89 04 24 e8 b9 f8 ff ff 48 8b 04
[  721.105370] RSP: 002b:00007fa92a540ce0 EFLAGS: 00000246 ORIG_RAX: 000000000000002c
[  721.106645] RAX: ffffffffffffffda RBX: 00007fa92a540db0 RCX: 00007fa92f0f5494
[  721.107845] RDX: 0000000000000002 RSI: 00007fa92adcce90 RDI: 0000000000000025
[  721.109724] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000
[  721.112061] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000
[  721.113593] R13: 0000000000000000 R14: 00007fa92a540db0 R15: 0000000000000001
[  721.114928] Modules linked in: tempesta_fw(OE) tempesta_db(OE) tempesta_tls(OE) tempesta_lib(OE) sha256_ssse3(E) sha512_ssse3(E) sha512_generic(E) uinput(E) xt_conntrack(E) nft_chain_nat(E) xt_MASQUERADE(E) nf_nat(E) nf_conntrack_netlink(E) nf_conntrack(E) nf_defrag_ipv6(E) nf_defrag_ipv4(E) xfrm_user(E) xfrm_algo(E) nft_counter(E) xt_addrtype(E) nft_compat(E) nf_tables(E) libcrc32c(E) nfnetlink(E) br_netfilter(E) bridge(E) stp(E) llc(E) overlay(E) hid_generic(E) usbhid(E) hid(E) rfkill(E) intel_rapl_msr(E) intel_rapl_common(E) intel_pmc_core(E) crc32_pclmul(E) ghash_clmulni_intel(E) sr_mod(E) cdrom(E) ata_generic(E) aesni_intel(E) snd_ens1371(E) snd_ac97_codec(E) libaes(E) crypto_simd(E) cryptd(E) ac97_bus(E) ehci_pci(E) uhci_hcd(E) glue_helper(E) gameport(E) snd_rawmidi(E) snd_seq_device(E) ehci_hcd(E) ata_piix(E) snd_pcm(E) vsock_loopback(E) usbcore(E) vmw_vsock_virtio_transport_common(E) snd_timer(E) e1000(E) libata(E) snd(E) vmw_balloon(E) joydev(E) vmw_vsock_vmci_transport(E)
[  721.114967]  pcspkr(E) soundcore(E) sg(E) vsock(E) i2c_piix4(E) vmw_vmci(E) button(E) ac(E) msr(E) parport_pc(E) ppdev(E) lp(E) parport(E) fuse(E) configfs(E) ip_tables(E) x_tables(E) autofs4(E) ext4(E) crc32c_generic(E) crc16(E) mbcache(E) jbd2(E) sd_mod(E) t10_pi(E) crc_t10dif(E) crct10dif_generic(E) vmwgfx(E) drm_kms_helper(E) cec(E) ttm(E) drm(E) mptspi(E) scsi_transport_spi(E) mptscsih(E) psmouse(E) mptbase(E) evdev(E) scsi_mod(E) crct10dif_pclmul(E) crct10dif_common(E) crc32c_intel(E) serio_raw(E) [last unloaded: tempesta_lib]
[  721.140046] ---[ end trace 47b7684c18b6f0fd ]---
[  721.140907] RIP: 0010:tdb_htrie_extend_rec+0x7a/0x90 [tempesta_db]
[  721.141997] Code: 75 1e 8b 4b 0c 85 c9 7e 21 48 c1 ea 07 f0 0f b1 53 08 85 c0 75 e5 48 83 c4 08 4c 89 c0 5b 5d c3 c1 e0 07 48 8d 5c 05 00 eb d1 <0f> 0b 48 83 c4 08 45 31 c0 4c 89 c0 5b 5d c3 0f 0b 0f 1f 44 00 00
[  721.145308] RSP: 0018:ffffa512404ec788 EFLAGS: 00010246
[  721.146221] RAX: 0000000000000000 RBX: ffff96218f301200 RCX: 0000000000000000
[  721.147450] RDX: 0000000000101280 RSI: 00000000fffffe01 RDI: ffffffffc0c5fb02
[  721.148855] RBP: ffff96218f200000 R08: ffff96218f301280 R09: 0000000000000000
[  721.150106] R10: ffffdc6180000000 R11: ffffa512404ec918 R12: ffffa512404ec890
[  721.151334] R13: 0000000000000000 R14: ffffffffc0d25840 R15: ffffa512404ec938
[  721.152648] FS:  00007fa92a542700(0000) GS:ffff9622da080000(0000) knlGS:0000000000000000
[  721.154042] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  721.155037] CR2: 00007fa92ea71bb0 CR3: 00000001826d6006 CR4: 0000000000370ee0
[  721.156373] Kernel panic - not syncing: Fatal exception in interrupt
[  721.157805] Kernel Offset: 0x1aa00000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff)
[  721.179344] Rebooting in 1 seconds..
[  722.181722] ACPI MEMORY or I/O RESET_REG.

Testing

Any test from cache.test_cache.