diff --git a/eBPF_Supermarket/kvm_watcher/Makefile b/eBPF_Supermarket/kvm_watcher/Makefile index 5264cb9a3..39fe08031 100644 --- a/eBPF_Supermarket/kvm_watcher/Makefile +++ b/eBPF_Supermarket/kvm_watcher/Makefile @@ -7,6 +7,7 @@ ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \ | sed 's/riscv64/riscv/' \ | sed 's/loongarch64/loongarch/') APP = src/kvm_watcher + OPTIONS = -f -w -n -d -e # 共同规则1 diff --git a/eBPF_Supermarket/kvm_watcher/docs/Hypercall.md b/eBPF_Supermarket/kvm_watcher/docs/Hypercall.md new file mode 100644 index 000000000..a855470d2 --- /dev/null +++ b/eBPF_Supermarket/kvm_watcher/docs/Hypercall.md @@ -0,0 +1,369 @@ +> 在Linux中，大家应该对syscall非常的了解和熟悉，其是用户态进入内核态的一种途径或者说是一种方式，完成了两个模式之间的切换；而在虚拟环境中，有没有一种类似于syscall这种方式，能够从no root模式切换到root模式呢？答案是肯定的，KVM提供了Hypercall机制，x86体系架构也有相关的指令支持。 +> +> hypercall：当虚拟机的Guest OS需要执行一些更高权限的操作（如：页表的更新、对物理资源的访问等）时，由于自身在非特权域无法完成这些操作，于是便通过调用Hypercall交给Hypervisor来完成这些操作。 + +## Hypercall的发起 + +KVM代码中提供了五种形式的Hypercall接口： + +``` +file: arch/x86/include/asm/kvm_para.h, line: 34 +static inline long kvm_hypercall0(unsigned int nr); +static inline long kvm_hypercall1(unsigned int nr, unsigned long p1); +static inline long kvm_hypercall2(unsigned int nr, unsigned long p1, unsigned long p2); +static inline long kvm_hypercall3(unsigned int nr, unsigned long p1, unsigned long p2, unsigned long p3) +static inline long kvm_hypercall4(unsigned int nr, unsigned long p1, unsigned long p2, unsigned long p3, unsigned long p4) +``` + +这几个接口的区别在于参数个数的不用，本质是一样的。挑个参数最多的看下： + +``` +static inline long kvm_hypercall4(unsigned int nr, unsigned long p1, + unsigned long p2, unsigned long p3, + unsigned long p4) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr), "b"(p1), "c"(p2), "d"(p3), "S"(p4) + : "memory"); + return ret; +} +``` + +Hypercall内部实现是标准的内嵌汇编,稍作分析： + +### KVM_HYPERCALL + +``` +#define KVM_HYPERCALL ".byte 0x0f,0x01,0xc1" +``` + +对于KVM hypercall来说，KVM_HYPERCALL是一个三字节的指令序列，x86体系架构下即是vmcall指令，官方手册解释： + +``` +vmcall： + op code:0F 01 C1 -- VMCALL Call to VM + monitor +by causing VM exit +``` + +言简意赅，vmcall会导致VM exit到VMM。 + +### 返回值 + +: “=a”(ret)，表示返回值放在eax寄存器中输出。 + +### 输入 + +: “a”(nr), “b”(p1), “c”(p2), “d”(p3), “S”(p4),表示输入参数放在对应的eax，ebx，ecx，edx，esi中，而nr其实就是可以认为是系统调用号。 + +## hypercall的处理 + +当Guest发起一次hypercall后，VMM会接管到该call导致的VM Exit。 + +``` +static int (*const kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { + ...... + [EXIT_REASON_VMCALL] = kvm_emulate_hypercall, + ...... +} +``` + +进入kvm_emulate_hypercall()处理，过程非常简单： + +``` +int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) +{ + unsigned long nr, a0, a1, a2, a3, ret; + int op_64_bit; + + // 检查是否启用了Xen超级调用，如果是，则调用Xen超级调用处理函数 + if (kvm_xen_hypercall_enabled(vcpu->kvm)) + return kvm_xen_hypercall(vcpu); + + // 检查是否启用了Hypervisor超级调用，如果是，则调用Hypervisor超级调用处理函数 + if (kvm_hv_hypercall_enabled(vcpu)) + return kvm_hv_hypercall(vcpu); + + // 从寄存器中读取超级调用号及参数 + nr = kvm_rax_read(vcpu); + a0 = kvm_rbx_read(vcpu); + a1 = kvm_rcx_read(vcpu); + a2 = kvm_rdx_read(vcpu); + a3 = kvm_rsi_read(vcpu); + + // 记录超级调用的追踪信息 + trace_kvm_hypercall(nr, a0, a1, a2, a3); + + // 检查是否为64位超级调用 + op_64_bit = is_64_bit_hypercall(vcpu); + if (!op_64_bit) { + nr &= 0xFFFFFFFF; + a0 &= 0xFFFFFFFF; + a1 &= 0xFFFFFFFF; + a2 &= 0xFFFFFFFF; + a3 &= 0xFFFFFFFF; + } + + // 检查当前CPU的特权级是否为0 + if (static_call(kvm_x86_get_cpl)(vcpu) != 0) { + ret = -KVM_EPERM; + goto out; + } + + ret = -KVM_ENOSYS; + + // 根据超级调用号执行相应的操作 + switch (nr) { + case KVM_HC_VAPIC_POLL_IRQ: + ret = 0; + break; + case KVM_HC_KICK_CPU: + // 处理CPU唤醒的超级调用 + if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT)) + break; + + kvm_pv_kick_cpu_op(vcpu->kvm, a1); + kvm_sched_yield(vcpu, a1); + ret = 0; + break; +#ifdef CONFIG_X86_64 + case KVM_HC_CLOCK_PAIRING: + // 处理时钟配对的超级调用 + ret = kvm_pv_clock_pairing(vcpu, a0, a1); + break; +#endif + case KVM_HC_SEND_IPI: + // 处理发送中断请求的超级调用 + if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI)) + break; + + ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit); + break; + case KVM_HC_SCHED_YIELD: + // 处理调度让出的超级调用 + if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD)) + break; + + kvm_sched_yield(vcpu, a0); + ret = 0; + break; + case KVM_HC_MAP_GPA_RANGE: + // 处理GPA范围映射的超级调用 + ret = -KVM_ENOSYS; + if (!(vcpu->kvm->arch.hypercall_exit_enabled & (1 << KVM_HC_MAP_GPA_RANGE))) + break; + + // 设置KVM_EXIT_HYPERCALL退出类型，并填充相关信息 + vcpu->run->exit_reason = KVM_EXIT_HYPERCALL; + vcpu->run->hypercall.nr = KVM_HC_MAP_GPA_RANGE; + vcpu->run->hypercall.args[0] = a0; + vcpu->run->hypercall.args[1] = a1; + vcpu->run->hypercall.args[2] = a2; + vcpu->run->hypercall.longmode = op_64_bit; + vcpu->arch.complete_userspace_io = complete_hypercall_exit; + return 0; + default: + ret = -KVM_ENOSYS; + break; + } + +out: + // 如果不是64位超级调用，则返回值需要截断为32位 + if (!op_64_bit) + ret = (u32)ret; + kvm_rax_write(vcpu, ret); + + // 更新超级调用统计信息，并跳过被模拟的指令 + ++vcpu->stat.hypercalls; + return kvm_skip_emulated_instruction(vcpu); +} +``` + +### Conclusion + +整个过程非常简洁和简单，hypercall机制给了Guest能够主动进入VMM的一种方式。 + +## 调用号 + +``` +#define KVM_HC_VAPIC_POLL_IRQ 1 +#define KVM_HC_MMU_OP 2 +#define KVM_HC_FEATURES 3 +#define KVM_HC_PPC_MAP_MAGIC_PAGE 4 +#define KVM_HC_KICK_CPU 5 +#define KVM_HC_MIPS_GET_CLOCK_FREQ 6 +#define KVM_HC_MIPS_EXIT_VM 7 +#define KVM_HC_MIPS_CONSOLE_OUTPUT 8 +#define KVM_HC_CLOCK_PAIRING 9 +#define KVM_HC_SEND_IPI 10 +#define KVM_HC_SCHED_YIELD 11 +#define KVM_HC_MAP_GPA_RANGE 12 +``` + + +1. ##### KVM_HC_VAPIC_POLL_IRQ + +------------------------ + +Architecture: x86 +Status: active +Purpose: 触发客户机退出，以便在重新进入时主机可以检查待处理的中断。 + +2. ##### KVM_HC_MMU_OP + +---------------- + +Architecture: x86 +Status: deprecated. +Purpose: 支持内存管理单元（MMU）操作，例如写入页表项（PTE）、刷新转换后备缓冲（TLB）以及释放页表（PT）。 + +3. ##### KVM_HC_FEATURES + +------------------ + +Architecture: PPC +Status: active +Purpose: 向客户机公开超级调用的可用性。在 x86 平台上，使用 cpuid 来列举可用的超级调用。在 PPC（PowerPC）上，可以使用基于设备树的查找（也是 EPAPR 规定的方式）或 KVM 特定的列举机制（即这个超级调用）。 + +4. ##### KVM_HC_PPC_MAP_MAGIC_PAGE + +---------------------------- + +Architecture: PPC +Status: active +Purpose:为了实现超级监视器与客户机之间的通信，存在一个共享页面，其中包含了监视器可见寄存器状态的部分。客户机可以通过使用此超级调用将这个共享页面映射，以通过内存访问其监视器寄存器。 + +5. ##### KVM_HC_KICK_CPU + +------------------ + +Architecture: x86 +Status: active +Purpose: 用于唤醒处于 HLT（Halt）状态的vCPU 。 +Usage example: +一个使用了半虚拟化的客户机的虚拟 CPU，在内核模式下忙等待某个事件的发生（例如，自旋锁变为可用）时，如果其忙等待时间超过了一个阈值时间间隔，就可以执行 HLT 指令。执行 HLT 指令将导致 hypervisor 将虚拟 CPU 置于休眠状态，直到发生适当的事件。同一客户机的另一个虚拟 CPU 可以通过发出 KVM_HC_KICK_CPU 超级调用来唤醒正在睡眠的虚拟 CPU，指定要唤醒的虚拟 CPU 的 APIC ID（a1）。另外一个参数（a0）在这个超级调用中用于将来的用途。 + + +6. ##### KVM_HC_CLOCK_PAIRING + +----------------------- + +Architecture: x86 +Status: active +Purpose: 用于同步主机和客户机时钟。 + +Usage: +a0：客户机物理地址，用于存储主机复制的 "struct kvm_clock_offset" 结构。 + +a1：时钟类型，目前只支持 KVM_CLOCK_PAIRING_WALLCLOCK（0）（对应主机的 CLOCK_REALTIME 时钟）。 + +```c +struct kvm_clock_pairing { + __s64 sec; // 从 clock_type 时钟起的秒数。 + __s64 nsec; // 从 clock_type 时钟起的纳秒数。 + __u64 tsc; // 用于计算 sec/nsec 对的客户机 TSC（时间戳计数）值。 + __u32 flags; // 标志，目前未使用（为 0）。 + __u32 pad[9]; // 填充字段，目前未使用。 +}; +``` + +这个超级调用允许客户机在主机和客户机之间计算精确的时间戳。客户机可以使用返回的 TSC（时间戳计数）值来计算其时钟的 CLOCK_REALTIME，即在同一时刻。 + +如果主机不使用 TSC 时钟源，或者时钟类型不同于 KVM_CLOCK_PAIRING_WALLCLOCK，则返回 KVM_EOPNOTSUPP。 + +7. ##### KVM_HC_SEND_IPI + +------------------ + +Architecture: x86 +Status: active +Purpose: 向多个vcpu发生ipi。 + +- `a0`: 目标 APIC ID 位图的低位部分。 +- `a1`: 目标 APIC ID 位图的高位部分。 +- `a2`: 位图中最低的 。 +- `a3`: 中断命令寄存器。 + +这个超级调用允许客户机发送组播中断处理请求（IPIs），每次调用最多可以有 128 个目标（在 64 位模式下）或者 64 个虚拟中央处理单元（vCPU）（在 32 位模式下）。目标由位图表示，位图包含在前两个参数中（a0 和 a1）。a0 的第 0 位对应于第三个参数 a2 中的 APIC ID，a0 的第 1 位对应于 a2+1 的 APIC ID，以此类推。 + +返回成功传递 IPIs 的 CPU 数量。 + +8. ##### KVM_HC_SCHED_YIELD + +--------------------- + +Architecture: x86 +Status: active +Purpose: 用于在目标vCPU被抢占时进行让步。 + +a0: destination APIC ID + +Usage example: 当向多个vCPU发送调用函数中断（call-function IPI）时，如果任何目标 vCPU 被抢占，进行让步。 + +9. ##### KVM_HC_MAP_GPA_RANGE + +------------------------- + +Architecture: x86 +Status: active +Purpose: 请求 KVM 映射一个具有指定属性的 GPA 范围。 + +`a0`: 起始页面的客户机物理地址 +`a1`: （4KB）页面的数量（在 GPA 空间中必须是连续的） +`a2`: 属性 + + 属性： + 位 3:0 - 首选页大小编码，0 = 4KB，1 = 2MB，2 = 1GB，以此类推... + 位 4 - 明文 = 0，加密 = 1 + 位 63:5 - 保留（必须为零） + +**实现注意事项** + +此超级调用通过 KVM_CAP_EXIT_HYPERCALL 能力在用户空间中实现。在向客户机 CPUID 中添加 KVM_FEATURE_HC_MAP_GPA_RANGE 之前，用户空间必须启用该能力。此外，如果客户机支持 KVM_FEATURE_MIGRATION_CONTROL，用户空间还必须设置一个 MSR 过滤器来处理对 MSR_KVM_MIGRATION_CONTROL 的写入。 + +可以通过如下查看发生的hypercall信息： + +``` +root@nans:/sys/kernel/debug/tracing/events/kvm# echo 0 > ../../tracing_on +root@nans:/sys/kernel/debug/tracing/events/kvm# echo 1 > kvm_hypercall/enable +root@nans:/sys/kernel/debug/tracing/events/kvm# echo 1 > ../../tracing_on +root@nans:/sys/kernel/debug/tracing/events/kvm# cat ../../trace_pipe +``` + +输出如下： + +![image-20240110125350965](https://gitee.com/nan-shuaibo/image/raw/master/202401101258714.png) + +使用ebpf技术统计hypercall信息： + +统计两秒内的每个hypercall发生的次数，和自客户机启动以来每个vcpu上发生的hypercall的次数 + +``` +------------------------------------------------------------------------ +TIME:16:22:05 +PID VCPU_ID NAME COUNTS HYPERCALLS +68453 4 KICK_CPU 1 0 +68453 2 KICK_CPU 1 0 +68453 1 SEND_IPI 6 5 +68453 0 SEND_IPI 7 7 +68453 7 KICK_CPU 1 0 +68453 0 KICK_CPU 1 0 +------------------------------------------------------------------------ +TIME:16:22:07 +PID VCPU_ID NAME COUNTS HYPERCALLS +68082 4 KICK_CPU 2 45 +68453 5 SEND_IPI 3 2 +68453 6 SCHED_YIELD 2 66 +68453 6 SEND_IPI 79 80 +68453 3 SEND_IPI 45 44 +68453 1 SEND_IPI 23 28 +68453 0 SEND_IPI 7 14 +68453 4 SEND_IPI 145 145 +``` + +并将详细信息输出至临时文件 + +![image-20240301162527679](https://gitee.com/nan-shuaibo/image/raw/master/202403011629545.png) + diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_hypercall.h b/eBPF_Supermarket/kvm_watcher/include/kvm_hypercall.h new file mode 100644 index 000000000..dba8bf0d8 --- /dev/null +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_hypercall.h @@ -0,0 +1,98 @@ +// Copyright 2023 The LMP Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/linuxkerneltravel/lmp/blob/develop/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: nanshuaibo811@163.com +// +// Kernel space BPF program used for monitoring data for KVM HYPERCALL. + +#ifndef __KVM_HYPERCALL_H +#define __KVM_HYPERCALL_H + +#include "kvm_watcher.h" +#include "vmlinux.h" +#include +#include +#include + +// 定义宏从寄存器读取超级调用信息 +// 代码来源：arch/x86/kvm/kvm_cache_regs.h +#define BUILD_KVM_GPR_ACCESSORS(lname, uname) \ + static __always_inline unsigned long kvm_##lname##_read( \ + struct kvm_vcpu *vcpu) { \ + return vcpu->arch.regs[VCPU_REGS_##uname]; \ + } + +BUILD_KVM_GPR_ACCESSORS(rax, RAX) +BUILD_KVM_GPR_ACCESSORS(rbx, RBX) +BUILD_KVM_GPR_ACCESSORS(rcx, RCX) +BUILD_KVM_GPR_ACCESSORS(rdx, RDX) +BUILD_KVM_GPR_ACCESSORS(rsi, RSI) + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1024); + __type(key, struct hc_key); + __type(value, struct hc_value); +} hc_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1024); + __type(key, struct hc_key); + __type(value, u32); +} hc_count SEC(".maps"); + +static int entry_emulate_hypercall(struct kvm_vcpu *vcpu, void *rb, + struct common_event *e, pid_t vm_pid) { + CHECK_PID(vm_pid); + u64 nr, a0, a1, a2, a3; + nr = kvm_rax_read(vcpu); // 超级调用号 + // 超级调用参数 + a0 = kvm_rbx_read(vcpu); + a1 = kvm_rcx_read(vcpu); + a2 = kvm_rdx_read(vcpu); + a3 = kvm_rsi_read(vcpu); + RESERVE_RINGBUF_ENTRY(rb, e); + e->process.pid = pid; + e->process.tid = (u32)bpf_get_current_pid_tgid(); + e->time = bpf_ktime_get_ns(); + bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm)); + e->hypercall_data.a0 = a0; + e->hypercall_data.a1 = a1; + e->hypercall_data.a2 = a2; + e->hypercall_data.a3 = a3; + e->hypercall_data.vcpu_id = vcpu->vcpu_id; + e->hypercall_data.hc_nr = nr; + e->hypercall_data.hypercalls = vcpu->stat.hypercalls; + bpf_ringbuf_submit(e, 0); + struct hc_key hc_key = {.pid = pid, .nr = nr, .vcpu_id = vcpu->vcpu_id}; + struct hc_value hc_value = {.a0 = a0, + .a1 = a1, + .a2 = a2, + .a3 = a3, + .counts = 1, + .hypercalls = vcpu->stat.hypercalls}; + u32 *count; + count = bpf_map_lookup_elem(&hc_count, &hc_key); + if (count) { + __sync_fetch_and_add(count, 1); + hc_value.counts = *count; + } else { + bpf_map_update_elem(&hc_count, &hc_key, &hc_value.counts, BPF_NOEXIST); + } + bpf_map_update_elem(&hc_map, &hc_key, &hc_value, BPF_ANY); + return 0; +} + +#endif /* __KVM_HYPERCALL_H */ \ No newline at end of file diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h index c388f0f2e..3a8a8df61 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h @@ -33,7 +33,7 @@ #define OUTPUT_INTERVAL(us) usleep((__u32)(us * MICROSECONDS_IN_SECOND)) -#define OPTIONS_LIST "-w, -p, -d, -f, -c, -i, or -e" +#define OPTIONS_LIST "-w, -p, -d, -f, -c, -i, ,-h or -e" #define PFERR_PRESENT_BIT 0 #define PFERR_WRITE_BIT 1 @@ -70,7 +70,7 @@ } while (0) // 定义清屏宏 -#define CLEAR_SCREEN() printf("\033[2J\033[H") +#define CLEAR_SCREEN() printf("\033[2J\033[H\n") #define RING_BUFFER_TIMEOUT_MS 100 @@ -88,11 +88,6 @@ return 0; \ } -struct ExitReason { - __u32 number; - const char *name; -}; - struct reason_info { __u64 time; __u64 reason; @@ -120,6 +115,22 @@ struct dirty_page_info { __u32 pid; }; +struct hc_value { + __u64 a0; + __u64 a1; + __u64 a2; + __u64 a3; + __u64 hypercalls; // vcpu上hypercall发生的次数 + __u32 counts; // 特定hypercall发生的次数 + __u32 pad; +}; + +struct hc_key { + __u64 nr; + pid_t pid; + __u32 vcpu_id; +}; + struct process { __u32 pid; __u32 tid; @@ -135,6 +146,7 @@ enum EventType { PAGE_FAULT, IRQCHIP, IRQ_INJECT, + HYPERCALL, } event_type; struct common_event { @@ -213,6 +225,17 @@ struct common_event { __u64 injections; // IRQ_INJECT 特有成员 } irq_inject_data; + + struct { + __u64 hc_nr; + __u64 a0; + __u64 a1; + __u64 a2; + __u64 a3; + __u64 hypercalls; + __u32 vcpu_id; + // HYPERCALL 特有成员 + } hypercall_data; }; }; diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c index b5c9e5462..9c32befae 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c @@ -24,6 +24,7 @@ #include "../include/kvm_vcpu.h" #include "../include/kvm_mmu.h" #include "../include/kvm_irq.h" +#include "../include/kvm_hypercall.h" #include "../include/kvm_watcher.h" char LICENSE[] SEC("license") = "Dual BSD/GPL"; @@ -138,4 +139,9 @@ int BPF_PROG(fentry_vmx_inject_irq, struct kvm_vcpu *vcpu, bool reinjected) { SEC("fexit/vmx_inject_irq") int BPF_PROG(fexit_vmx_inject_irq, struct kvm_vcpu *vcpu, bool reinjected) { return exit_vmx_inject_irq(vcpu, &rb, e); -} \ No newline at end of file +} + +SEC("fentry/kvm_emulate_hypercall") +int BPF_PROG(fentry_emulate_hypercall, struct kvm_vcpu *vcpu) { + return entry_emulate_hypercall(vcpu, &rb, e, vm_pid); +} diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 3c11e0b3a..089c045f5 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -30,162 +30,127 @@ #include "../include/kvm_watcher.h" #include "kvm_watcher.skel.h" -// 定义具体的退出原因 arch/x86/include/uapi/asm/vmx.h -struct ExitReason exitReasons[] = {{0, "EXCEPTION_NMI"}, - {1, "EXTERNAL_INTERRUPT"}, - {2, "TRIPLE_FAULT"}, - {3, "INIT_SIGNAL"}, - {4, "SIPI_SIGNAL"}, - {7, "INTERRUPT_WINDOW"}, - {8, "NMI_WINDOW"}, - {9, "TASK_SWITCH"}, - {10, "CPUID"}, - {12, "HLT"}, - {13, "INVD"}, - {14, "INVLPG"}, - {15, "RDPMC"}, - {16, "RDTSC"}, - {18, "VMCALL"}, - {19, "VMCLEAR"}, - {20, "VMLAUNCH"}, - {21, "VMPTRLD"}, - {22, "VMPTRST"}, - {23, "VMREAD"}, - {24, "VMRESUME"}, - {25, "VMWRITE"}, - {26, "VMOFF"}, - {27, "VMON"}, - {28, "CR_ACCESS"}, - {29, "DR_ACCESS"}, - {30, "IO_INSTRUCTION"}, - {31, "MSR_READ"}, - {32, "MSR_WRITE"}, - {33, "INVALID_STATE"}, - {34, "MSR_LOAD_FAIL"}, - {36, "MWAIT_INSTRUCTION"}, - {37, "MONITOR_TRAP_FLAG"}, - {39, "MONITOR_INSTRUCTION"}, - {40, "PAUSE_INSTRUCTION"}, - {41, "MCE_DURING_VMENTRY"}, - {43, "TPR_BELOW_THRESHOLD"}, - {44, "APIC_ACCESS"}, - {45, "EOI_INDUCED"}, - {46, "GDTR_IDTR"}, - {47, "LDTR_TR"}, - {48, "EPT_VIOLATION"}, - {49, "EPT_MISCONFIG"}, - {50, "INVEPT"}, - {51, "RDTSCP"}, - {52, "PREEMPTION_TIMER"}, - {53, "INVVPID"}, - {54, "WBINVD"}, - {55, "XSETBV"}, - {56, "APIC_WRITE"}, - {57, "RDRAND"}, - {58, "INVPCID"}, - {59, "VMFUNC"}, - {60, "ENCLS"}, - {61, "RDSEED"}, - {62, "PML_FULL"}, - {63, "XSAVES"}, - {64, "XRSTORS"}, - {67, "UMWAIT"}, - {68, "TPAUSE"}, - {74, "BUS_LOCK"}, - {75, "NOTIFY"}}; +// 创建并打开临时文件 +FILE *create_temp_file(const char *filename) { + const char *directory = "./temp"; + char filepath[256]; -const char *getExitReasonName(int number) { - for (int i = 0; i < sizeof(exitReasons) / sizeof(exitReasons[0]); i++) { - if (exitReasons[i].number == number) { - return exitReasons[i].name; - } - } - return "Unknown"; // 如果找不到对应的退出原因，返回一个默认值 -} + // 构建文件的完整路径 + snprintf(filepath, sizeof(filepath), "%s/%s", directory, filename); -typedef struct { - int exit_reason; - char info[256]; - unsigned long long total_dur; - unsigned long long avg_dur; -} ExitInfo; - -// 链表节点 -typedef struct Node { - ExitInfo data; - struct Node *next; -} Node; - -Node *exitInfoBuffer = NULL; - -void addExitInfo(Node **head, int exit_reason, const char *info, - unsigned long long dur, int count) { - Node *newNode = (Node *)malloc(sizeof(Node)); - newNode->data.exit_reason = exit_reason; - strncpy(newNode->data.info, info, sizeof(newNode->data.info)); - newNode->next = NULL; - newNode->data.total_dur = dur; - newNode->data.avg_dur = dur / count; - - // 检查是否已经存在相同 exit reason 的信息 - Node *current = *head; - Node *previous = NULL; - while (current != NULL) { - if (current->data.exit_reason == exit_reason) { - // 更新已存在的信息 - strncpy(current->data.info, info, sizeof(current->data.info)); - current->data.total_dur = dur + current->data.total_dur; - current->data.avg_dur = current->data.total_dur / count; - free(newNode); // 释放新节点，因为信息已经更新 - return; - } - previous = current; - current = current->next; + // 创建目录，如果不存在 + if (mkdir(directory, 0777) == -1 && errno != EEXIST) { + perror("Failed to create directory"); + return NULL; } - // 没有找到相同的 exit reason，将新节点添加到链表 - if (previous != NULL) { - previous->next = newNode; - } else { - *head = newNode; + + // 尝试打开文件 + FILE *output = fopen(filepath, "w"); + if (!output) { + perror("Failed to open output file"); + return NULL; } + + return output; } -// 查找指定退出原因的信息 -const char *findExitInfo(Node *head, int exit_reason) { - Node *current = head; - while (current != NULL) { - if (current->data.exit_reason == exit_reason) { - return current->data.info; +const char *getHypercallName(int number) { + struct Hypercall { + int number; + const char *name; + }; + + // 定义超级调用 include\uapi\linux\kvm_para.h + struct Hypercall hypercalls[] = { + {1, "VAPIC_POLL_IRQ"}, {5, "KICK_CPU"}, {9, "CLOCK_PAIRING"}, + {10, "SEND_IPI"}, {11, "SCHED_YIELD"}, {12, "MAP_GPA_RANGE"}}; + + for (int i = 0; i < sizeof(hypercalls) / sizeof(hypercalls[0]); i++) { + if (hypercalls[i].number == number) { + return hypercalls[i].name; } - current = current->next; } - return NULL; + return "Unknown"; // 如果找不到对应的超级调用号，返回一个默认值 } -// 释放链表 -void freeExitInfoList(Node *head) { - while (head != NULL) { - Node *temp = head; - head = head->next; - free(temp); - } -} -// 打印退出的信息 -void printExitInfo(Node *head) { - Node *current = head; - CLEAR_SCREEN(); - printf( - "-----------------------------------------------------------------" - "----------\n"); - printf("%-21s %-18s %-8s %-8s %-13s \n", "EXIT_REASON", "COMM", "PID", - "COUNT", "AVG_DURATION(us)"); - while (current != NULL) { - printf("%-2d/%-18s %-33s %-13.4f \n", current->data.exit_reason, - getExitReasonName(current->data.exit_reason), current->data.info, - NS_TO_US_WITH_DECIMAL(current->data.avg_dur)); - current = current->next; +const char *getExitReasonName(int number) { + struct ExitReason { + int number; + const char *name; + }; + + // 定义具体的退出原因 arch/x86/include/uapi/asm/vmx.h + struct ExitReason exitReasons[] = {{0, "EXCEPTION_NMI"}, + {1, "EXTERNAL_INTERRUPT"}, + {2, "TRIPLE_FAULT"}, + {3, "INIT_SIGNAL"}, + {4, "SIPI_SIGNAL"}, + {7, "INTERRUPT_WINDOW"}, + {8, "NMI_WINDOW"}, + {9, "TASK_SWITCH"}, + {10, "CPUID"}, + {12, "HLT"}, + {13, "INVD"}, + {14, "INVLPG"}, + {15, "RDPMC"}, + {16, "RDTSC"}, + {18, "VMCALL"}, + {19, "VMCLEAR"}, + {20, "VMLAUNCH"}, + {21, "VMPTRLD"}, + {22, "VMPTRST"}, + {23, "VMREAD"}, + {24, "VMRESUME"}, + {25, "VMWRITE"}, + {26, "VMOFF"}, + {27, "VMON"}, + {28, "CR_ACCESS"}, + {29, "DR_ACCESS"}, + {30, "IO_INSTRUCTION"}, + {31, "MSR_READ"}, + {32, "MSR_WRITE"}, + {33, "INVALID_STATE"}, + {34, "MSR_LOAD_FAIL"}, + {36, "MWAIT_INSTRUCTION"}, + {37, "MONITOR_TRAP_FLAG"}, + {39, "MONITOR_INSTRUCTION"}, + {40, "PAUSE_INSTRUCTION"}, + {41, "MCE_DURING_VMENTRY"}, + {43, "TPR_BELOW_THRESHOLD"}, + {44, "APIC_ACCESS"}, + {45, "EOI_INDUCED"}, + {46, "GDTR_IDTR"}, + {47, "LDTR_TR"}, + {48, "EPT_VIOLATION"}, + {49, "EPT_MISCONFIG"}, + {50, "INVEPT"}, + {51, "RDTSCP"}, + {52, "PREEMPTION_TIMER"}, + {53, "INVVPID"}, + {54, "WBINVD"}, + {55, "XSETBV"}, + {56, "APIC_WRITE"}, + {57, "RDRAND"}, + {58, "INVPCID"}, + {59, "VMFUNC"}, + {60, "ENCLS"}, + {61, "RDSEED"}, + {62, "PML_FULL"}, + {63, "XSAVES"}, + {64, "XRSTORS"}, + {67, "UMWAIT"}, + {68, "TPAUSE"}, + {74, "BUS_LOCK"}, + {75, "NOTIFY"}}; + + for (int i = 0; i < sizeof(exitReasons) / sizeof(exitReasons[0]); i++) { + if (exitReasons[i].number == number) { + return exitReasons[i].name; + } } + return "Unknown"; // 如果找不到对应的退出原因，返回一个默认值 } + // 检查具有给定 PID 的进程是否存在 int doesVmProcessExist(pid_t pid) { char proc_name[256]; @@ -228,25 +193,12 @@ int compare(const void *a, const void *b) { // 保存脏页信息到文件 int save_count_dirtypagemap_to_file(struct bpf_map *map) { - const char *directory = "./temp"; - const char *filename = "./temp/dirty_temp"; - - // 创建目录，如果不存在 - if (mkdir(directory, 0777) == -1) { - // 如果目录已经存在，这里的错误是预期的，可以忽略 - // 否则，打印错误信息并返回 - if (errno != EEXIST) { - perror("Failed to create directory"); - return -1; - } - } - - FILE *output = fopen(filename, "w"); + const char *filename = "dirty_temp"; + FILE *output = create_temp_file(filename); if (!output) { - perror("Failed to open output file"); - return -1; + fprintf(stderr, "Failed to create file in directory\n"); + return 1; } - int count_dirty_fd = bpf_map__fd(map); struct dirty_page_info lookup_key = {}; struct dirty_page_info next_key = {}; @@ -311,6 +263,7 @@ static struct env { bool mmio_page_fault; bool execute_irqchip; bool execute_irq_inject; + bool execute_hypercall; int monitoring_time; pid_t vm_pid; enum EventType event_type; @@ -324,6 +277,7 @@ static struct env { .execute_irqchip = false, .execute_irq_inject = false, .mmio_page_fault = false, + .execute_hypercall = false, .monitoring_time = 0, .vm_pid = -1, .event_type = NONE_TYPE, @@ -343,24 +297,23 @@ static const struct argp_option opts[] = { "Monitor virtual machine dirty page information."}, {"kvmmmu_page_fault", 'f', NULL, 0, "Monitoring the data of kvmmmu page fault."}, - {"kvm_irqchip", 'c', NULL, 0, + {"kvm_irqchip(software)", 'c', NULL, 0, "Monitor the irqchip setting information in KVM VM."}, - {"irq_inject(x86)", 'i', NULL, 0, + {"irq_inject(hardware)", 'i', NULL, 0, "Monitor the virq injection information in KVM VM "}, - {"stat", 's', NULL, 0, - "Display statistical data.(The -e option must be specified.)"}, + {"hypercall", 'h', NULL, 0, "Monitor the hypercall information in KVM VM "}, {"mmio", 'm', NULL, 0, "Monitoring the data of mmio page fault.(The -f option must be " "specified.)"}, {"vm_pid", 'p', "PID", 0, "Specify the virtual machine pid to monitor."}, {"monitoring_time", 't', "SEC", 0, "Time for monitoring."}, - {NULL, 'h', NULL, OPTION_HIDDEN, "Show the full help"}, + {NULL, 'H', NULL, OPTION_HIDDEN, "Show the full help"}, {}, }; // 解析命令行参数 static error_t parse_arg(int key, char *arg, struct argp_state *state) { switch (key) { - case 'h': + case 'H': argp_state_help(state, stderr, ARGP_HELP_STD_HELP); break; case 'w': @@ -387,6 +340,9 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) { case 'i': SET_OPTION_AND_CHECK_USAGE(option_selected, env.execute_irq_inject); break; + case 'h': + SET_OPTION_AND_CHECK_USAGE(option_selected, env.execute_hypercall); + break; case 's': if (env.execute_exit) { env.ShowStats = true; @@ -467,6 +423,8 @@ static int determineEventType(struct env *env) { env->event_type = IRQCHIP; } else if (env->execute_irq_inject) { env->event_type = IRQ_INJECT; + } else if (env->execute_hypercall) { + env->event_type = HYPERCALL; } else { env->event_type = NONE_TYPE; // 或者根据需要设置一个默认的事件类型 } @@ -625,6 +583,44 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { e->irq_inject_data.soft ? "Soft/INTn" : "IRQ"); break; } + case HYPERCALL: { + const char *filename = "./temp/hc_temp"; + FILE *output = fopen(filename, "a"); + if (!output) { + perror("Failed to open output file"); + return -1; + } + fprintf(output, "%-18.6f %-15s %-10d %-10d %-10s %-11llu", + timestamp_ms, e->process.comm, e->process.pid, + e->hypercall_data.vcpu_id, + getHypercallName(e->hypercall_data.hc_nr), + e->hypercall_data.hypercalls); + if (e->hypercall_data.hc_nr == 5) { + fprintf(output, "apic_id:%llu\n", e->hypercall_data.a1); + } else if (e->hypercall_data.hc_nr == 9) { + fprintf( + output, "GPA:%#llx CLOCK_TYPE:%s\n", e->hypercall_data.a0, + e->hypercall_data.a1 == 0 ? "KVM_CLOCK_PAIRING_WALLCLOCK" + : ""); + } else if (e->hypercall_data.hc_nr == 10) { + fprintf(output, + "ipi_bitmap_low:%#llx,ipi_bitmap_high:%#llx,min(apic_" + "id):%llu,icr:%#llx\n", + e->hypercall_data.a0, e->hypercall_data.a1, + e->hypercall_data.a2, e->hypercall_data.a3); + } else if (e->hypercall_data.hc_nr == 11) { + fprintf(output, "dest apic_id:%llu\n", e->hypercall_data.a0); + } else if (e->hypercall_data.hc_nr == 12) { + fprintf(output, + "GPA start:%#llx,PAGE_NR(4KB):%llu,Attributes:%#llx\n", + e->hypercall_data.a0, e->hypercall_data.a1, + e->hypercall_data.a2); + } else { + fprintf(output, "\n"); + } + fclose(output); + break; + } default: // 处理未知事件类型 break; @@ -645,9 +641,6 @@ static int print_event_head(struct env *env) { "VAILD?"); break; case EXIT: - // printf("%-18s %-21s %-18s %-15s %-8s %-13s \n", "TIME(ms)", - // "EXIT_REASON", "COMM", "PID/TID", "COUNT", - // "DURATION(us)"); break; case HALT_POLL: printf("%-18s %-15s %-15s %-10s %-7s %-11s %-10s\n", "TIME(ms)", @@ -672,6 +665,19 @@ static int print_event_head(struct env *env) { "TIME(ms)", "COMM", "PID", "DELAY", "IRQ_NR", "VCPU_ID", "INJECTIONS", "TYPE"); break; + case HYPERCALL: { + const char *filename = "hc_temp"; + FILE *output = create_temp_file(filename); + if (!output) { + fprintf(stderr, "Failed to create file in directory\n"); + return 1; + } + fprintf(output, "%-18s %-15s %-10s %-10s %-10s %-10s %-10s\n", + "TIME(ms)", "COMM", "PID", "VCPU_ID", "NAME", "HYPERCALLS", + "ARGS"); + fclose(output); + break; + } default: // Handle default case or display an error message break; @@ -717,6 +723,67 @@ static void set_disable_load(struct kvm_watcher_bpf *skel) { env.execute_irq_inject ? true : false); bpf_program__set_autoload(skel->progs.fexit_vmx_inject_irq, env.execute_irq_inject ? true : false); + bpf_program__set_autoload(skel->progs.fentry_emulate_hypercall, + env.execute_hypercall ? true : false); +} + +int print_hc_map(struct kvm_watcher_bpf *skel) { + int fd = bpf_map__fd(skel->maps.hc_map); + int count_fd = bpf_map__fd(skel->maps.hc_count); + int err; + struct hc_key lookup_key = {}; + struct hc_key next_key = {}; + struct hc_value hc_value = {}; + struct tm *tm; + char ts[32]; + time_t t; + time(&t); + tm = localtime(&t); + strftime(ts, sizeof(ts), "%H:%M:%S", tm); + int first_run = 1; + // Iterate over the map + while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { + if (first_run) { + first_run = 0; + printf( + "--------------------------------------------------------------" + "----------" + "\n"); + printf("TIME:%s\n", ts); + printf("%-12s %-12s %-12s %-12s %-12s\n", "PID", "VCPU_ID", "NAME", + "COUNTS", "HYPERCALLS"); + } + // Print the current entry + err = bpf_map_lookup_elem(fd, &next_key, &hc_value); + if (err < 0) { + fprintf(stderr, "failed to lookup hc_value: %d\n", err); + return -1; + } + printf("%-12d %-12d %-12s %-12d %-12lld\n", next_key.pid, + next_key.vcpu_id, getHypercallName(next_key.nr), hc_value.counts, + hc_value.hypercalls); + // // Move to the next key + lookup_key = next_key; + } + memset(&lookup_key, 0, sizeof(struct hc_key)); + while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { + err = bpf_map_delete_elem(fd, &next_key); + if (err < 0) { + fprintf(stderr, "failed to cleanup hc_map: %d\n", err); + return -1; + } + lookup_key = next_key; + } + memset(&lookup_key, 0, sizeof(struct hc_key)); + while (!bpf_map_get_next_key(count_fd, &lookup_key, &next_key)) { + err = bpf_map_delete_elem(count_fd, &next_key); + if (err < 0) { + fprintf(stderr, "failed to cleanup hc_count: %d\n", err); + return -1; + } + lookup_key = next_key; + } + return 0; } int print_exit_map(struct kvm_watcher_bpf *skel) { @@ -771,6 +838,14 @@ int print_exit_map(struct kvm_watcher_bpf *skel) { return 0; } +void print_map_and_check_error(int (*print_func)(struct kvm_watcher_bpf *), struct kvm_watcher_bpf *skel, const char *map_name, int err) { + OUTPUT_INTERVAL(OUTPUT_INTERVAL_SECONDS); + print_func(skel); + if (err < 0) { + printf("Error printing %s map: %d\n", map_name, err); + } +} + int main(int argc, char **argv) { // 定义一个环形缓冲区 struct ring_buffer *rb = NULL; @@ -844,8 +919,13 @@ int main(int argc, char **argv) { } while (!exiting) { err = ring_buffer__poll(rb, RING_BUFFER_TIMEOUT_MS /* timeout, ms */); - sleep(3); - err = print_exit_map(skel); + + if (env.execute_hypercall) { + print_map_and_check_error(print_hc_map, skel, "hypercall", err); + } + if (env.execute_exit) { + print_map_and_check_error(print_exit_map, skel, "exit", err); + } /* Ctrl-C will cause -EINTR */ if (err == -EINTR) { err = 0; @@ -856,10 +936,7 @@ int main(int argc, char **argv) { break; } } - if (env.ShowStats) { - printExitInfo(exitInfoBuffer); - freeExitInfoList(exitInfoBuffer); - } else if (env.execute_mark_page_dirty) { + if (env.execute_mark_page_dirty) { err = save_count_dirtypagemap_to_file(skel->maps.count_dirty_map); if (err < 0) { printf("Save count dirty page map to file fail: %d\n", err);