TDB: kernel panic in tfw_client_ent_init

[RomanBelozerov]

Scope

Tempesta commit - 9061800

Tempesta config:

cache 0;
cache_fulfill * *;
cache_methods GET HEAD;
cache_ttl 3600;

listen 443 proto=https;
listen 80;

srv_group default {server ${server_ip}:8000;}

srv_group auth {server ${server_ip}:8000;}

tls_certificate ${tempesta_workdir}/tempesta.crt;
tls_certificate_key ${tempesta_workdir}/tempesta.key;
tls_match_any_server_name;

req_hdr_add X-Forwarded-Proto "https";
resp_hdr_set Strict-Transport-Security "max-age=31536000; includeSubDomains";

vhost tempesta-tech.com {
        resp_hdr_set Cache-Control;
        proxy_pass default;
}

vhost default {
        proxy_pass default;
        nonidempotent GET * *;
        nonidempotent HEAD * *;
        nonidempotent POST * *;
}

vhost auth {proxy_pass auth;}

vhost wp-auth {
        req_hdr_add Location "wp-admin";
        proxy_pass auth;
}

frang_limits {
        http_method_override_allowed true;
        http_methods post put get;
}

cache_purge;
cache_purge_acl ${client_ip};

block_action attack reply;
block_action error reply;

http_chain admin {
        mark == 23  -> default;
        -> auth;
}

http_chain {
	# Set cache_ttl for speciefic locations
	uri == "/blog/*" -> cache_ttl = 1;
    uri == "/knowledge-base/*" -> cache_ttl = 600;
        
	# Set Security
	mark == 23 -> default;
    uri == "*/google/*" -> auth;
    uri == "*/wp-admin/*" -> admin;
    uri == "*/wp-login.php*" -> admin;
    uri == "*/phpmyadmin*" -> admin;
    -> default;
}

Trace:

[  921.058511] [tdb] ERROR: out of free space
[  921.059406] [tdb] ERROR: Cannot allocate cache entry for key=0xc645a56c23a0a57c
[  921.060098] BUG: kernel NULL pointer dereference, address: 0000000000000168
[  921.060736] #PF: supervisor write access in kernel mode
[  921.061279] #PF: error_code(0x0002) - not-present page
[  921.061814] PGD 0 P4D 0 
[  921.062257] Oops: 0002 [#1] SMP NOPTI
[  921.062791] CPU: 3 PID: 10632 Comm: python3 Tainted: G           OE     5.10.35.tfw-2678723 #1
[  921.063672] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.15.0-1 04/01/2014
[  921.064568] RIP: 0010:tfw_client_ent_init+0x3e/0x180 [tempesta_fw]
[  921.065556] Code: be 08 01 00 00 53 48 89 fb 48 83 c7 38 48 83 ec 40 65 48 8b 04 25 28 00 00 00 48 89 45 e0 31 c0 48 b8 ff ff ff ff ff ff ff 7f <c7> 87 30 01 00 00 00 00 00 00 48 89 87 28 01 00 00 e8 3c 3f d6 ff
[  921.067404] RSP: 0018:ffffb8fdc0150308 EFLAGS: 00010246
[  921.068094] RAX: 7fffffffffffffff RBX: 0000000000000000 RCX: 0000000000000027
[  921.068870] RDX: 0000000000000000 RSI: 0000000000000108 RDI: 0000000000000038
[  921.069605] RBP: ffffb8fdc0150360 R08: ffff9fa070ed8a40 R09: ffffb8fdc0150148
[  921.070345] R10: 0000000000000001 R11: 0000000000000001 R12: ffffb8fdc0150400
[  921.071148] R13: 0000000000000008 R14: ffffb8fdc01503d0 R15: 0000000000000000
[  921.071885] FS:  00007f1b9b7a8700(0000) GS:ffff9fa070ec0000(0000) knlGS:0000000000000000
[  921.072648] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  921.073310] CR2: 0000000000000168 CR3: 000000002e74c006 CR4: 0000000000770ee0
[  921.074329] PKRU: 55555554
[  921.074969] Call Trace:
[  921.075576]  <IRQ>
[  921.076095]  ? printk+0x58/0x6f
[  921.076637]  tdb_rec_get_alloc+0xfd/0x120 [tempesta_db]
[  921.077292]  tfw_client_obtain+0x18a/0x260 [tempesta_fw]
[  921.077962]  ? tfw_client_obtain+0x260/0x260 [tempesta_fw]
[  921.078657]  ? tfw_cfg_exit+0x10/0x10 [tempesta_fw]
[  921.079327]  tfw_http_req_client_link+0x191/0x1f0 [tempesta_fw]
[  921.079990]  tfw_http_req_process+0x1de/0x930 [tempesta_fw]
[  921.080619]  ? alloc_pages_current+0x87/0xe0
[  921.081245]  ? __get_free_pages+0x11/0x40
[  921.081912]  ? __tfw_pool_new+0x24/0x70 [tempesta_fw]
[  921.082574]  ? bzero_fast+0xe/0x10 [tempesta_lib]
[  921.083213]  ? bzero_fast+0xe/0x10 [tempesta_lib]
[  921.083827]  ? tfw_http_init_parser_req+0x22/0x50 [tempesta_fw]
[  921.084516]  tfw_http_msg_process_generic+0x18c/0x6e0 [tempesta_fw]
[  921.085209]  ? __skb_ext_put+0xa7/0xd0
[  921.085786]  tfw_http_msg_process+0x3e/0x60 [tempesta_fw]
[  921.086444]  tfw_connection_recv+0x87/0x110 [tempesta_fw]
[  921.087170]  ? ss_skb_unroll+0xce/0x270 [tempesta_fw]
[  921.087837]  ss_tcp_process_data+0x1ec/0x3f0 [tempesta_fw]
[  921.088525]  ss_tcp_data_ready+0x44/0xe0 [tempesta_fw]
[  921.089211]  tcp_data_ready+0x2b/0xd0
[  921.089781]  tcp_data_queue+0x452/0xd10
[  921.090350]  tcp_rcv_established+0x230/0x670
[  921.090945]  ? sk_filter_trim_cap+0xde/0x240
[  921.091562]  tcp_v4_do_rcv+0x140/0x200
[  921.092100]  tcp_v4_rcv+0xcfd/0xe10
[  921.092618]  ip_protocol_deliver_rcu+0x30/0x1b0
[  921.093188]  ip_local_deliver_finish+0x48/0x60
[  921.093752]  ip_local_deliver+0xfa/0x110
[  921.094285]  ? ip_protocol_deliver_rcu+0x1b0/0x1b0
[  921.094907]  ip_rcv_finish+0x87/0xa0
[  921.095473]  ip_sabotage_in+0x5b/0x70 [br_netfilter]
[  921.096055]  nf_hook_slow+0x45/0xb0
[  921.096586]  ip_rcv+0x90/0xe0
[  921.097057]  ? ip_rcv_finish_core.isra.0+0x420/0x420
[  921.097635]  __netif_receive_skb_one_core+0x88/0xa0
[  921.098195]  __netif_receive_skb+0x18/0x60
[  921.098704]  netif_receive_skb+0x49/0x150
[  921.099228]  br_netif_receive_skb+0x36/0x50 [bridge]
[  921.099772]  br_pass_frame_up+0x143/0x160 [bridge]
[  921.100302]  ? fdb_find_rcu+0xed/0x170 [bridge]
[  921.100819]  br_handle_frame_finish+0x182/0x460 [bridge]
[  921.101406]  ? fib_table_lookup+0x234/0x670
[  921.101912]  ? br_pass_frame_up+0x160/0x160 [bridge]
[  921.102458]  br_nf_hook_thresh+0xcb/0xe0 [br_netfilter]
[  921.103047]  ? br_pass_frame_up+0x160/0x160 [bridge]
[  921.103640]  br_nf_pre_routing_finish+0x142/0x320 [br_netfilter]
[  921.104231]  ? br_pass_frame_up+0x160/0x160 [bridge]
[  921.104786]  ? nf_nat_ipv4_fn+0x4f/0x70 [nf_nat]
[  921.105317]  ? nf_nat_ipv4_pre_routing+0x2a/0x70 [nf_nat]
[  921.105892]  br_nf_pre_routing+0x254/0x540 [br_netfilter]
[  921.106465]  ? br_nf_forward_ip+0x4a0/0x4a0 [br_netfilter]
[  921.107060]  br_handle_frame+0x1b2/0x370 [bridge]
[  921.107599]  ? br_pass_frame_up+0x160/0x160 [bridge]
[  921.108135]  ? br_handle_frame_finish+0x460/0x460 [bridge]
[  921.108716]  __netif_receive_skb_core+0x2d5/0xf60
[  921.109240]  ? ip_rcv_finish+0x87/0xa0
[  921.109739]  ? ip_rcv+0xcc/0xe0
[  921.110191]  __netif_receive_skb_one_core+0x3f/0xa0
[  921.110740]  __netif_receive_skb+0x18/0x60
[  921.111257]  process_backlog+0xa9/0x160
[  921.111731]  net_rx_action+0x13e/0x390
[  921.112194]  __do_softirq+0xd9/0x291
[  921.112661]  asm_call_irq_on_stack+0xf/0x20
[  921.113146]  </IRQ>
[  921.113542]  do_softirq_own_stack+0x3d/0x50
[  921.114024]  do_softirq.part.0+0x46/0x50
[  921.114516]  __local_bh_enable_ip+0x50/0x60
[  921.115006]  ip_finish_output2+0x1ab/0x590
[  921.115500]  ? __cgroup_bpf_run_filter_skb+0x3c3/0x3d0
[  921.116032]  __ip_finish_output+0xd8/0x220
[  921.116523]  ip_finish_output+0x2d/0xb0
[  921.117037]  ip_output+0x7a/0x100
[  921.117491]  ? __ip_finish_output+0x220/0x220
[  921.117998]  ip_local_out+0x3d/0x50
[  921.118456]  __ip_queue_xmit+0x17a/0x470
[  921.118940]  ip_queue_xmit+0x15/0x20
[  921.119417]  __tcp_transmit_skb+0xa20/0xbe0
[  921.119910]  ? ip_local_out+0x3d/0x50
[  921.120377]  tcp_write_xmit+0x2cc/0x11b0
[  921.120871]  __tcp_push_pending_frames+0x37/0x100
[  921.121395]  tcp_push+0xfc/0x100
[  921.121836]  tcp_sendmsg_locked+0xd36/0xe70
[  921.122328]  tcp_sendmsg+0x2d/0x50
[  921.122778]  inet_sendmsg+0x43/0x70
[  921.123241]  sock_sendmsg+0x5e/0x70
[  921.123699]  __sys_sendto+0x113/0x190
[  921.124181]  ? __cgroup_bpf_run_filter_getsockopt+0xb0/0x2f0
[  921.124747]  ? __secure_computing+0x42/0xe0
[  921.125240]  __x64_sys_sendto+0x29/0x30
[  921.125708]  do_syscall_64+0x38/0x90
[  921.126161]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
[  921.126691] RIP: 0033:0x7f1b9d88193e
[  921.127155] Code: 49 89 f4 55 53 89 fb 48 83 ec 10 e8 6c f8 ff ff 45 31 c9 45 31 c0 45 89 f2 89 c5 4c 89 ea 4c 89 e6 89 df b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 36 89 ef 48 89 44 24 08 e8 9e f8 ff ff 48 8b
[  921.128652] RSP: 002b:00007f1b9b7a7530 EFLAGS: 00000246 ORIG_RAX: 000000000000002c
[  921.129342] RAX: ffffffffffffffda RBX: 0000000000000005 RCX: 00007f1b9d88193e
[  921.130003] RDX: 00000000000002c1 RSI: 00007f1b9400fe50 RDI: 0000000000000005
[  921.130668] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000
[  921.131356] R10: 0000000000000000 R11: 0000000000000246 R12: 00007f1b9400fe50
[  921.132027] R13: 00000000000002c1 R14: 0000000000000000 R15: 00007f1b9d151753
[  921.132705] Modules linked in: tempesta_fw(OE) tempesta_db(OE) tempesta_tls(OE) tempesta_lib(OE) tcp_diag inet_diag sha256_ssse3 sha512_ssse3 xt_nat xt_tcpudp veth tls xt_conntrack nft_chain_nat xt_MASQUERADE nf_nat nf_conntrack_netlink nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 xfrm_user xfrm_algo nft_counter xt_addrtype nft_compat nf_tables libcrc32c nfnetlink br_netfilter bridge stp llc overlay snd_hda_codec_generic ledtrig_audio snd_hda_intel snd_intel_dspcfg snd_hda_codec intel_rapl_msr intel_rapl_common snd_hda_core snd_hwdep binfmt_misc snd_pcm kvm_intel nls_iso8859_1 kvm snd_seq_midi crct10dif_pclmul ghash_clmulni_intel snd_seq_midi_event aesni_intel qxl crypto_simd snd_rawmidi drm_ttm_helper cryptd ttm snd_seq glue_helper drm_kms_helper snd_seq_device snd_timer cec joydev input_leds snd mac_hid fb_sys_fops syscopyarea sysfillrect soundcore sysimgblt serio_raw qemu_fw_cfg sch_fq_codel msr parport_pc ppdev lp parport ramoops reed_solomon drm efi_pstore ip_tables x_tables autofs4
[  921.132752]  hid_generic usbhid hid virtio_net crc32_pclmul net_failover lpc_ich psmouse ahci i2c_i801 i2c_smbus libahci failover virtio_rng virtio_blk [last unloaded: tempesta_lib]
[  921.141636] CR2: 0000000000000168
[  921.142201] ---[ end trace 72f0357e977a16cf ]---
[  921.142864] RIP: 0010:tfw_client_ent_init+0x3e/0x180 [tempesta_fw]
[  921.143604] Code: be 08 01 00 00 53 48 89 fb 48 83 c7 38 48 83 ec 40 65 48 8b 04 25 28 00 00 00 48 89 45 e0 31 c0 48 b8 ff ff ff ff ff ff ff 7f <c7> 87 30 01 00 00 00 00 00 00 48 89 87 28 01 00 00 e8 3c 3f d6 ff
[  921.145440] RSP: 0018:ffffb8fdc0150308 EFLAGS: 00010246
[  921.146175] RAX: 7fffffffffffffff RBX: 0000000000000000 RCX: 0000000000000027
[  921.146997] RDX: 0000000000000000 RSI: 0000000000000108 RDI: 0000000000000038
[  921.147796] RBP: ffffb8fdc0150360 R08: ffff9fa070ed8a40 R09: ffffb8fdc0150148
[  921.148566] R10: 0000000000000001 R11: 0000000000000001 R12: ffffb8fdc0150400
[  921.149397] R13: 0000000000000008 R14: ffffb8fdc01503d0 R15: 0000000000000000
[  921.150205] FS:  00007f1b9b7a8700(0000) GS:ffff9fa070ec0000(0000) knlGS:0000000000000000
[  921.151113] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  921.151860] CR2: 0000000000000168 CR3: 000000002e74c006 CR4: 0000000000770ee0
[  921.152666] PKRU: 55555554
[  921.153233] Kernel panic - not syncing: Fatal exception in interrupt
[  921.154147] Kernel Offset: 0x1b400000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff)
[  921.155238] Rebooting in 1 seconds..
[  922.147919] ACPI MEMORY or I/O RESET_REG.

Testing

Test to reproduce - t_stress.test_ddos.TestDDoSL7.test_ddos_post_method from 438 with 8 docker containers and cmd python3 /app/start.py GET http://192.168.122.116:80 1 1 /app/files/proxies/http.txt 1 10