kvm_watcher:在vm exit基础上添加kvm userspace exit退出原因统计功能

eBPF_Supermarket/kvm_watcher/README.md

@@ -55,7 +55,8 @@ BPF program used for monitoring KVM event
   -c, --kvm_irqchip          Monitor the irqchip setting information in KVM
                              VM.
   -d, --mark_page_dirty      Monitor virtual machine dirty page information.
-  -e, --vm_exit              Monitoring the event of vm exit.
+  -e, --vm_exit              Monitoring the event of vm exit(including exiting
+                             to KVM and user mode).
   -f, --kvmmmu_page_fault    Monitoring the data of kvmmmu page fault.
   -h, --hypercall            Monitor the hypercall information in KVM VM 
   -i, --irq_inject           Monitor the virq injection information in KVM VM 
@@ -145,7 +146,7 @@ graph TD;
   要运行测试，请执行以下命令：
 
   ```
-  make test
+  make deps test
   ```
 
   这将自动执行上述测试流程，并在结束后提供测试结果。

eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md

@@ -1,6 +1,6 @@
 # kvm_exit
 
-考虑到频繁的虚拟机退出事件可能会导致性能问题，kvm_watcher中的kvm_exit子功能通过显示详细的退出原因和在一台主机上运行的所有vm的每个虚拟机的vcpu上的退出计数及处理时延，可以捕获和分析vm exit事件，该工具旨在定位频繁退出的原因（如EPT_VIOLATION、EPT_MISCONFIG、PML_FULL等）。
+考虑到频繁的虚拟机退出事件可能会导致性能问题，kvm_watcher中的kvm_exit子功能通过显示详细的退出原因和在一台主机上运行的所有vm的每个虚拟机的vcpu上的退出计数及处理时延，可以捕获和分析vm exit事件，该工具旨在定位频繁退出的原因（如EPT_VIOLATION、EPT_MISCONFIG、PML_FULL等）,在vm exit基础上，如果kvm这个时候因为某些原因，需要退出到用户态的hypervisor(比如qemu)，kvm就要设置KVM_EXIT_XXX，此工具包含了这两部分exit reason。
 
 ## 原理介绍
 
@@ -30,14 +30,18 @@
 
 ![VM entry 与 VM exit](https://ctf-wiki.org/pwn/virtualization/basic-knowledge/figure/vm-entry-and-exit.png)
 
+### kvm_exit与kvm_userspace_exit
 
+[vm exit和userspace exit](https://blog.csdn.net/weixin_46324627/article/details/136325212?spm=1001.2014.3001.5501)
 
 ## 挂载点
 
-| 类型       | 名称      |
-| ---------- | --------- |
-| tracepoint | kvm_exit  |
-| tracepoint | kvm_entry |
+| 类型       | 名称                    |
+| ---------- | ----------------------- |
+| tracepoint | kvm_exit                |
+| tracepoint | kvm_entry               |
+| fentry     | kvm_arch_vcpu_ioctl_run |
+| tracepoint | kvm_userspace_exit      |
 
 ## 示例输出
 
@@ -98,4 +102,5 @@ pid          tid          total_time   max_time     min_time     counts       re
 - **VM Exit 原因统计**：记录并展示触发 VM Exit 的具体原因，帮助用户理解 VM Exit 发生的上下文和背景。
 - **VM Exit 延时分析**：统计每次 VM Exit 处理的最大、最小和总共延时，为性能分析提供量化数据。
 - **VM Exit 次数计数**：计算每种类型的 VM Exit 发生的次数，帮助识别最频繁的性能瓶颈。
-- **PID、TID号**：其中PID为主机侧的虚拟机进程号，TID为虚拟机内部的vcpu**的进程号**
+- **PID、TID号**：其中PID为主机侧的虚拟机进程号，TID为虚拟机内部的vcpu**的进程号**
+

eBPF_Supermarket/kvm_watcher/include/kvm_exits.h

@@ -30,18 +30,33 @@
 struct {
     __uint(type, BPF_MAP_TYPE_HASH);
     __uint(max_entries, 8192);
-    __type(key, struct exit_key);      // exit_key:reason pid pad[2]
+    __type(key, struct exit_key);      // exit_key:reason pid tid
     __type(value, struct exit_value);  // exit_value : max_time total_time
                                        // min_time count pad
 } exit_map SEC(".maps");
 
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(max_entries, 8192);
+    __type(key, struct exit_key);      // exit_key:reason pid tid
+    __type(value, struct exit_value);  // exit_value : max_time total_time
+                                       // min_time count pad
+} userspace_exit_map SEC(".maps");
+
 struct {
     __uint(type, BPF_MAP_TYPE_HASH);
     __uint(max_entries, 8192);
     __type(key, pid_t);
     __type(value, struct reason_info);  // reason_info:time、reason、count
 } times SEC(".maps");
 
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(max_entries, 8192);
+    __type(key, pid_t);
+    __type(value, u64);  // reason_info:time、reason、count
+} userspace_exit_times SEC(".maps");
+
 struct exit {
     u64 pad;
     unsigned int exit_reason;
@@ -53,6 +68,11 @@ struct exit {
     u32 error_code;
     unsigned int vcpu_id;
 };
+struct userspace_exit {
+	u64 pad;
+	u32 reason;
+	int errno;
+};
 
 static int trace_kvm_exit(struct exit *ctx) {
     u32 reason;
@@ -72,6 +92,28 @@ static int trace_kvm_exit(struct exit *ctx) {
     return 0;
 }
 
+static void update_exit_map(void *map, struct exit_key *key, u64 duration_ns) {
+    struct exit_value *exit_value;
+    exit_value = bpf_map_lookup_elem(map, key);
+    if (exit_value) {
+        exit_value->count++;
+        exit_value->total_time += duration_ns;
+        if (exit_value->max_time < duration_ns) {
+            exit_value->max_time = duration_ns;
+        }
+        if (exit_value->min_time > duration_ns) {
+            exit_value->min_time = duration_ns;
+        }
+    } else {
+        struct exit_value new_exit_value = {.count = 1,
+                                            .max_time = duration_ns,
+                                            .total_time = duration_ns,
+                                            .min_time = duration_ns};
+        bpf_map_update_elem(map, key, &new_exit_value, BPF_ANY);
+    }
+}
+
+
 static int trace_kvm_entry() {
     struct reason_info *reas;
     pid_t pid, tid;
@@ -90,24 +132,33 @@ static int trace_kvm_entry() {
     exit_key.pid = pid;
     exit_key.tid = tid;
     exit_key.reason = reas->reason;
-    struct exit_value *exit_value;
-    exit_value = bpf_map_lookup_elem(&exit_map, &exit_key);
-    if (exit_value) {
-        exit_value->count++;
-        exit_value->total_time += duration_ns;
-        if (exit_value->max_time < duration_ns) {
-            exit_value->max_time = duration_ns;
-        }
-        if (exit_value->min_time > duration_ns) {
-            exit_value->min_time = duration_ns;
-        }
-    } else {
-        struct exit_value new_exit_value = {.count = 1,
-                                            .max_time = duration_ns,
-                                            .total_time = duration_ns,
-                                            .min_time = duration_ns};
-        bpf_map_update_elem(&exit_map, &exit_key, &new_exit_value, BPF_ANY);
+    update_exit_map(&exit_map, &exit_key, duration_ns);
+    return 0;
+}
+
+static int trace_kvm_userspace_entry(struct kvm_vcpu *vcpu){
+    pid_t tid = (u32)bpf_get_current_pid_tgid();
+    u64 ts = bpf_ktime_get_ns();
+    bpf_map_update_elem(&userspace_exit_times,&tid,&ts,BPF_ANY);
+    return 0;
+}
+static int trace_kvm_userspace_exit(struct userspace_exit *ctx){
+    pid_t tid = (u32)bpf_get_current_pid_tgid();
+    pid_t pid = bpf_get_current_pid_tgid() >> 32;
+    u64 *start_ts,ts,duration_ns;
+    start_ts=bpf_map_lookup_elem(&userspace_exit_times, &tid);
+    if (!start_ts || ctx->errno < 0) {
+        return 0;
     }
+    duration_ns = bpf_ktime_get_ns() - *start_ts;
+    bpf_map_delete_elem(&userspace_exit_times, &tid);
+    struct exit_key exit_key;
+    __builtin_memset(&exit_key, 0, sizeof(struct exit_key));
+    exit_key.pid = pid;
+    exit_key.tid = tid;
+    exit_key.reason = ctx->reason;
+    update_exit_map(&userspace_exit_map, &exit_key, duration_ns);
     return 0;
 }
+
 #endif /* __KVM_EXITS_H */

eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h

@@ -32,7 +32,7 @@
 
 #define OUTPUT_INTERVAL(SECONDS) sleep(SECONDS)
 
-#define OPTIONS_LIST "-w, -p, -d, -f, -c, -i, ,-h or -e"
+#define OPTIONS_LIST "-w, -d, -f, -c, -i, -l , -o , -h or -e"
 
 #define PFERR_PRESENT_BIT 0
 #define PFERR_WRITE_BIT 1
@@ -145,6 +145,13 @@ enum EventType {
     IOCTL,
 } event_type;
 
+enum NameType {
+    UNKNOWN_NAME_TYPE,
+    HYPERCALL_NR,
+    EXIT_NR,
+    EXIT_USERSPACE_NR,
+} name_type;
+
 struct common_event {
     struct process process;
     __u64 time;

eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c

@@ -67,7 +67,7 @@ SEC("tp/kvm/kvm_entry")
 int tp_entry(struct exit *ctx) {
     return trace_kvm_entry();
 }
-//记录VCPU调度的信息
+// 记录VCPU调度的信息
 SEC("kprobe/vmx_vcpu_load")
 int BPF_KPROBE(kp_vmx_vcpu_load, struct kvm_vcpu *vcpu, int cpu) {
     return trace_vmx_vcpu_load(vcpu, cpu, &rb, e);
@@ -163,5 +163,16 @@ int BPF_PROG(fentry_emulate_hypercall, struct kvm_vcpu *vcpu) {
 
 SEC("tp/syscalls/sys_enter_ioctl")
 int tp_ioctl(struct trace_event_raw_sys_enter *args) {
+    CHECK_PID(vm_pid);
     return trace_kvm_ioctl(args);
+}
+SEC("fentry/kvm_arch_vcpu_ioctl_run")
+int BPF_PROG(fentry_kvm_arch_vcpu_ioctl_run, struct kvm_vcpu *vcpu) {
+    CHECK_PID(vm_pid);
+    return trace_kvm_userspace_entry(vcpu);
+}
+
+SEC("tp/kvm/kvm_userspace_exit")
+int tp_kvm_userspace_exit(struct userspace_exit *ctx) {
+    return trace_kvm_userspace_exit(ctx);
 }