From c5a286cab9d4839aeff5715d5a103c4b352d63aa Mon Sep 17 00:00:00 2001 From: Hongyu Ning Date: Wed, 27 Sep 2023 10:53:45 +0800 Subject: [PATCH 1/8] [New]: add guest-test framework folder and description add guest-test framework folder and description Signed-off-by: Hongyu Ning --- guest-test/README.md | 119 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 guest-test/README.md diff --git a/guest-test/README.md b/guest-test/README.md new file mode 100644 index 00000000..275c1ae3 --- /dev/null +++ b/guest-test/README.md @@ -0,0 +1,119 @@ +# KVM/QEMU based Guest VM auto test framework + +## Description +A simple auto guest VM test framework for several types of VM launched by KVM/QEMU. +Types of VM supported includes: legacy, tdx, tdxio, +may extend in future for specific type of VMs, +in fact, the main differences among above VM types are QEMU config parameters. + +As QEMU config parameters may vary from version to version, +current implementation is QEMU version 7.2.0 based with tdx, tdxio special feature support. + +The qemu.config.json and common test framework code need to change along with new QEMU update + +## Limitaion +Each test execution will launch a new VM and run specific test scripts/binaries in guest VM, +test log will be captured from QEMU launching VM to test scripts/binaries execution in guest VM, +untill VM launched being shutdown properly or pkilled on purpose in abnormal test status. + +No multi-VMs test scenarios covered/supported. + +In any case of issue debugging, please refer to above test log with VM QEMU config info +and launch VM and debug issues manually running test scripts/binaries. + +A prepared Guest OS Image (qcow2 or raw image format) is requried with preset root account and password, +several values/parameters in qemu.config.json highly depend on Guest OS Image, please accomodate accordingly. + +## Usage +### qemu.config.json description +with QEMU emulator version 7.2 support, qemu.config.json parameters are fully aligned to it, +and grouped in 4 by 1st-level-keys: "common", "vm", "tdx", "tdxio" + +group "common" includes all configurable values to be passed to group "vm", "tdx", "tdxio" +2nd-level-keys info: +"kernel_img": [mandatory] /abs/path/to/vmlinuz file or bzImage file of target VM guest kernel +"initrd_img": [optional] /abs/path/to/initrd file or initramfs file of target VM guest kernel +"bios_img": [legacy vm optional, tdx/tdxio vm mandatory] /abs/path/to/ovmf file or other bios file of target VM guest bios +"qemu_img": [mandatory] /abs/path/to/qemu-kvm or qemu-system-x86_64 file to boot target VM guest +"guest_img": [mandatory] /abs/path/to/VM guest OS image with qcow2 format or raw image format +"guest_img_format": [mandatory] value range in [qcow2/raw], guest os image file type qcow2 or raw image +"boot_pattern": [mandatory] Guest OS booting pattern shows bootup completed, depends on Guest OS image provided +"guest_root_passwd": [mandatory] Guest OS root account password +"vm_type": [mandatory] value range in [legacy/tdx/tdxio], VM type to test includes legacy vm, tdx vm or tdxio vm +"pmu": [mandatory] value range in [on/off], qemu config -cpu pmu=on or -cpu pmu=off +"cpus": [mandatory] value range in [1 ~ maximum vcpu number], qemu config -smp cpus=$VCPU +"sockets": [mandatory] value range in [1 ~ maximum sockets number], qemu config -smp sockets=$SOCKETS +"mem": [mandatory] value range in [1 ~ maximum mem size in GB], qemu config -m memory size in GB +"cmdline": [optional] value range in [guest kernel paramter extra string], qemu config -append extra command line parameters to pass to VM guest kernel +"debug": [mandatory] value range in [on/off], qemu config -object tdx-guest,debug=on or -object tdx-guest,debug=off + +group "vm" includes all legacy vm launch qemu config options, which can be used to launch legacy vm standalone or as base part to launch tdx vm +group "vm" 2nd-level-keys could be bypassed if not provided (file not exists) +"cfg_var_6" & "cfg_var_10" + +group "tdx" includes tdx vm specific qemu config options, which is used to launch tdx vm (with group "vm" as base) or as a based part to launch tdxio vm + +group "tdxio" includes tdxio vm specific qemu config options, which is used to launch tdxio vm (with group "vm" + "tdx" as base) + +note about qemu.config.json: +- no changes allowed on 1st-level-keys hierarchy +- cfg_x part is allowed to extend freely based on needs +- cfg_var_x part is allowed to revised/extended too, please remember to keep changes specifically aligned in qemu_get_config.py + +### guest.test_launcher description +main test entrance, with following key args can be passed to override the values in qemu.config.json + -v $VCPU number of vcpus + -s $SOCKETS number of sockets + -m $MEM memory size in GB + -d $DEBUG debug on/off + -t $VM_TYPE vm_type legacy/tdx/tdxio + -x $TESTCASE testcase pass to test_executor + -c $CMDLINE guest kernel extra commandline + -p $PMU guest pmu off/on + -g $GCOV code coverage test mode off/on + +above key args will be recorded in a fresh new test_params.py for further import/source purpose accross scripts + +by enter each test, qemu_get_config.py will be called to get following pre-set parameters from qemu.config.json + $KERNEL_IMG values passed by group "common" key "kernel_img" + $INITRD_IMG values passed by group "common" key "initrd_img" + $BIOS_IMG values passed by group "common" key "bios_img" + $QEMU_IMG values passed by group "common" key "qemu_img" + $GUEST_IMG values passed by group "common" key "guest_img" + $GUEST_IMG_FORMAT values passed by group "common" key "guest_img_format" + $BOOT_PATTERN values passed by group "common" key "boot_pattern" + $SSHPASS values passed by group "common" key "guest_root_passwd" + +call guest.qemu_runner.sh and wait for $BOOT_PATTERN (shows VM boot up completed and ready for login) during VM boot + $BOOT_PATTERN selected based on following CentOS Stream 8/9 boot log example: "*Kernel*on*x86_64*" + boot log quoted: + CentOS Stream 9 + Kernel 6.5.0-rc5-next-20230809-next-20230809 on an x86_64 + Activate the web console with: systemctl enable --now cockpit.socket + CentOS-9 login: + +if $BOOT_PATTERN found, call guest.test_executor.sh with proper $TESTCASE to be executed in VM Guest, and shutdown VM after test compelted +if $BOOT_PATTERN not found, several VM life-cycles management logic applied to handle boot failure in different stages +if $ERR_STRx found, handle the error info accordingly (err_handlers) +no matter what, in the end, pkill VM process to avoid any potential test step failures above + +Note: bydeault, $GCOV is off, if $GCOV is on, above VM life-cycles management logic will be bypassed to keep VM process alive for gcov code coverage data collection + +### guest.qemu_runner description +VM boot engine, with parames exported from qemu_get_config.py and test scenario config sourced from test_params.py + +before VM boot, for $VM_TYPE tdx or tdxio, tdx_pre_check will be called to make sure basic environment is ready for TDX/TDXIO launching +VM boot is triggered by qemu_runner.py based on $VM_TYPE, with proper qemu config options applied + +### guest.test_executor description +guest VM test execution basic framework implemented in guerst.test_executor.sh, such as + guest_test_prepare, function based on sshpass to scp common.sh and test_script.sh to Guest VM + guest_test_source_code, function based on sshpass to scp source_code_dir and compile test_binary in Guest VM + guest_test_entry, function based on sshpass to execute test_script.sh and potential script params in Guest VM + guest_test_close, function based on sshpass to close VM + +## How to add new feature test +as described above, if simply add new TCs to run based on current qemu.config.json format, just need to implement it in test_executor with new $TESTCASE branch, +common functions of test_executor should be good enough to prepare/run/close new $TESTCASE + +if qemu.config.json format will be revised due to feature changes on QEMU implementation, please update qemu.config.json and qemu_get_config.py accordingly From fa3f817103d677a249ca1bf165606f676e6c7089 Mon Sep 17 00:00:00 2001 From: Hongyu Ning Date: Wed, 27 Sep 2023 15:10:56 +0800 Subject: [PATCH 2/8] [New][guest-test]: add test_launcher script test_launcher script initial commit Signed-off-by: Hongyu Ning --- guest-test/guest.test_launcher.sh | 315 ++++++++++++++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100755 guest-test/guest.test_launcher.sh diff --git a/guest-test/guest.test_launcher.sh b/guest-test/guest.test_launcher.sh new file mode 100755 index 00000000..ebdda31d --- /dev/null +++ b/guest-test/guest.test_launcher.sh @@ -0,0 +1,315 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (c) 2023 Intel Corporation + +# Author: Hongyu Ning +# +# History: 24, Aug., 2023 - Hongyu Ning - creation + + +# @desc This script is top level of Guest VM test +# @ PART 0: prepare test prerequisites +# @ PART 1: get params from qemu_get_config.py and script args, generate test_params.py +# @ PART 2: launch qemu_runner along with test_executor/err_handlers +# @ PART 3: err_handlers +# @ PART 4: timeout control in case of tdvm boot up failure/test failure +# @ PART 5: clean up at test execution ends + +###################### Variables ###################### +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" +echo "$SCRIPT_DIR" + +# code coverage test mode on/off, default off +# can be override by -g parameter +# in case of on, keep VM alive for gcov data collection +GCOV="off" +# timeout control in case of TD VM booting hang +SECONDS=0 +TIMEOUT=60 +# EXEC_FLAG=0 shows test_executor being called +EXEC_FLAG=1 + +# ERR_STR and ERR_FLAG definition +# for any unexpected error/warning/call trace handling +# unchecked MSR access error +ERR_STR1="*unchecked MSR access error*" +ERR_FLAG1=0 +# unexpected #VE error +ERR_STR2="*Unexpected #VE*" +ERR_FLAG2=0 +# general BUG info +ERR_STR3="*BUG:*" +ERR_FLAG3=0 +# general WARNING info +ERR_STR4="*WARNING:*" +ERR_FLAG4=0 +# general Call Trace info +ERR_STR5="*Call Trace:*" +ERR_FLAG5=0 +# + +###################### Functions ###################### +# helper function +usage() { + cat <<-EOF +NOTE!! args passed here will override params in qemu.confg.json + usage: ./${0##*/} + -v number of vcpus + -s number of sockets + -m memory size in GB + -d debug on/off + -t vm_type legacy/tdx/tdxio + -x testcase pass to test_executor + -c guest kernel extra commandline + -p guest pmu off/on + -g [optional, default off] code coverage test mode off/on + -h HELP info +EOF +} + +###################### Do Works ###################### +cd "$(dirname "$0")" 2>/dev/null || exit 1 +source ../.env + +## PART 0: prepare test prerequisites ## +which sshpass > /dev/null +if [ $? -ne 0 ]; then + dnf install -y sshpass > /dev/null + apt install -y sshpass > /dev/null +else + test_print_trc "sshpass prerequisites is ready for use" + test_print_trc "VM test is starting now..." +fi + +## PART 1: get params from qemu.cfg.json and script args ## +# generate a unified random port number in one test cycle +PORT=$(shuf -i 10010-10900 -n 1) +# generate test_params.py based on script args +echo PORT=$PORT > $SCRIPT_DIR/test_params.py + +# 1.1 get test scenario config related params from script args +# append following params in above fresh new test_params.py +# used across test_launcher.sh, qemu_runner.py, test_executor.sh + +# get args for QEMU boot configurable parameters +while getopts :v:s:m:d:t:x:c:p:g:h arg; do + case $arg in + v) + VCPU=$OPTARG + echo VCPU=$VCPU >> $SCRIPT_DIR/test_params.py + ;; + s) + SOCKETS=$OPTARG + echo SOCKETS=$SOCKETS >> $SCRIPT_DIR/test_params.py + ;; + m) + MEM=$OPTARG + echo MEM=$MEM >> $SCRIPT_DIR/test_params.py + ;; + d) + DEBUG=$OPTARG + echo DEBUG=\"$DEBUG\" >> $SCRIPT_DIR/test_params.py + ;; + t) + VM_TYPE=$OPTARG + echo VM_TYPE=\"$VM_TYPE\" >> $SCRIPT_DIR/test_params.py + ;; + x) + TESTCASE=$OPTARG + echo TESTCASE=\"$TESTCASE\" >> $SCRIPT_DIR/test_params.py + ;; + c) + CMDLINE=$OPTARG + echo CMDLINE=\"$CMDLINE\" >> $SCRIPT_DIR/test_params.py + ;; + p) + PMU=$OPTARG + echo PMU=\"$PMU\" >> $SCRIPT_DIR/test_params.py + ;; + g) + GCOV=$OPTARG + echo GCOV=\"$GCOV\" >> $SCRIPT_DIR/test_params.py + ;; + :) + test_print_err "Must supply an argument to -$OPTARG." + usage && exit 1 + ;; + \?) + test_print_err "Invalid Option -$OPTARG ignored." + usage && exit 1 + ;; + esac +done + + +# 1.2 get general parameter config from qemu.cfg.json +# all general parameter exported for qemu_runner and test_executor +# $KERNEL_IMG $INITRD_IMG $BIOS_IMG $QEMU_IMG $GUEST_IMG +# $GUEST_IMG_FORMAT $BOOT_PATTERN $SSHPASS $PORT + +#global_variable +output=$(python3 $SCRIPT_DIR/qemu_get_config.py) +KERNEL_IMG=$(echo $output | awk '{print $1}') +INITRD_IMG=$(echo $output | awk '{print $2}') +BIOS_IMG=$(echo $output | awk '{print $3}') +QEMU_IMG=$(echo $output | awk '{print $4}') +GUEST_IMG=$(echo $output | awk '{print $5}') +GUEST_IMG_FORMAT=$(echo $output | awk '{print $6}') +BOOT_PATTERN=$(echo $output | awk '{print $7}') +SSHPASS=$(echo $output | awk '{print $8}') + +test_print_trc "KERNEL_IMG $KERNEL_IMG" +test_print_trc "INITRD_IMG $INITRD_IMG" +test_print_trc "BIOS_IMG $BIOS_IMG" +test_print_trc "QEMU_IMG $QEMU_IMG" +test_print_trc "GUEST_IMG $GUEST_IMG" +test_print_trc "GUEST_IMG_FORMAT $GUEST_IMG_FORMAT" +test_print_trc "BOOT_PATTERN $BOOT_PATTERN" +test_print_trc "SSHPASS $SSHPASS" +test_print_trc "PORT $PORT" + +export KERNEL_IMG +export INITRD_IMG +export BIOS_IMG +export QEMU_IMG +export GUEST_IMG +export GUEST_IMG_FORMAT +export BOOT_PATTERN +export SSHPASS +export PORT +export GCOV + +## PART 2: launch qemu_runner ## +# launch qemu_runner along with err_handlers/test_executor +# 2.1 boot TD VM via qemu_runner with params +# exported from qemu_get_config.py & sourced from test_params.py +# 2.2 check TD VM boot $BOOT_PATTERN, then run test by test_executor +# 2.3 check TD VM boot $ERR_STRs, then run corresponding err_handler ($ERR_FLAGs) +# 2.4 break while loop if reach $TIMEOUT seconds (in case of TD VM boot hang) + +cd $SCRIPT_DIR +rm -rf /root/.ssh/known_hosts +while read -r line; do + echo "[${VM_TYPE}_vm]: $line" + # within $TIMEOUT but bypass the very first 2 seconds to avoid unexpected $BOOT_PATTERN match (from parameter handling logic) + if [[ $SECONDS -lt $TIMEOUT ]] && [[ $SECONDS -ge 2 ]]; then + if [[ $line == $BOOT_PATTERN ]]; then + test_print_trc "VM_TYPE: $VM_TYPE, VCPU: $VCPU, SOCKETS: $SOCKETS, MEM: $MEM, DEBUG: $DEBUG, PMU: $PMU, CMDLINE: $CMDLINE, TESTCASE: $TESTCASE, SECONDS: $SECONDS" + EXEC_FLAG=0 + ./guest.test_executor.sh + if [ $? -ne 0 ]; then # break while read loop in case of TD VM test failure + break + fi + # err_handlers string matching + elif [[ $line == $ERR_STR1 ]]; then + test_print_err "There is $ERR_STR1, test is not fully PASS" + ERR_FLAG1=1 + elif [[ $line == $ERR_STR2 ]]; then + test_print_err "There is $ERR_STR2, test failed" + ERR_FLAG2=1 + elif [[ $line == $ERR_STR3 ]] && [[ $line != *"DEBUG"* ]]; then + test_print_wrg "There is $ERR_STR3, please check" + ERR_FLAG3=1 + elif [[ $line == $ERR_STR4 ]]; then + test_print_wrg "There is $ERR_STR4, please check" + ERR_FLAG4=1 + elif [[ $line == $ERR_STR5 ]]; then + test_print_wrg "There is $ERR_STR5, please check" + ERR_FLAG5=1 + fi + # end of err_handlers string matching + elif [[ $SECONDS -ge $TIMEOUT ]]; then # break while read loop in case of TD VM boot timeout (no $BOOT_PATTERN found) + break + fi +done < <(if [ $GCOV == "off" ]; then timeout $TIMEOUT ./guest.qemu_runner.sh; else ./guest.qemu_runner.sh; fi) + +## PART 3: err_handlers error management +# unexpected error/bug/warning/call trace handling +if [ $ERR_FLAG1 -ne 0 ]; then + die "$VM_TYPE VM test failed with $ERR_STR1, please check |ERROR| in test log for more info" +fi + +if [ $ERR_FLAG2 -ne 0 ]; then + die "$VM_TYPE VM test failed with $ERR_STR2, please check |ERROR| in test log for more info" +fi + +if [ $ERR_FLAG3 -ne 0 ]; then + test_print_wrg "$VM_TYPE VM test hit $ERR_STR3, please check |WARNING| in test log for more info" +fi + +if [ $ERR_FLAG4 -ne 0 ]; then + test_print_wrg "$VM_TYPE VM test hit $ERR_STR4, please check |WARNING| in test log for more info" +fi + +if [ $ERR_FLAG5 -ne 0 ]; then + test_print_wrg "$VM_TYPE VM test hit $ERR_STR5, please check |WARNING| in test log for more info" +fi +# end of err_handlers error management + +## PART 4: timeout control in case of tdvm boot up failure/test failure ## +# sleep 3 seconds before starting VM life-cycles management logic +sleep 3 + +# VM life-cycles management step 1 +# check if TDVM is still up, non-zero return value indicates TDVM is not accessible +# TDVM not acccessible cases: +# a. TDVM is already closed after test +# b. TDVM boot up stuck at some point +sshpass -e ssh -p "$PORT" -o StrictHostKeyChecking=no root@localhost << EOF + echo "$VM_TYPE VM guest kernel under test:" + uname -r +EOF + +# VM life-cycles management step 2 +# non-zero return value of TD VM not accessible handling +# time count between 3 and $TIMEOUT is expected case a +# - handling: nothing to do, since TDVM is closed after test +# time count great or equal than $TIMEOUT is case b +# - handling: kill the tdvm_$PORT process since it's stuck +# time count less or qual than 3 is case b +# - handling: nothing to do, die for TDVM boot early failure, likely qemu config issue +if [ $? -ne 0 ]; then + if [ $SECONDS -gt 3 ] && [ $SECONDS -lt $TIMEOUT ] && [ $EXEC_FLAG -eq 0 ]; then + test_print_trc "$VM_TYPE VM test complete..." + elif [ $SECONDS -ge $TIMEOUT ] && [ $GCOV == "on" ]; then + pkill "${VM_TYPE}vm_$PORT" + die "TEST TIMEOUT!!!!!!!!!!!!" + elif [ $GCOV == "off" ]; then + die "$VM_TYPE VM test seems fail at beginning, please check test log" + fi +# zero return value shows TDVM is still accessible handling +# handing: no matter why it's still accessible, close it by ssh login +elif [ $GCOV == "off" ]; then + sshpass -e ssh -p "$PORT" -o StrictHostKeyChecking=no root@localhost << EOF + echo "$VM_TYPE VM guest kernel under test:" + uname -r + systemctl reboot --reboot-argument=now +EOF + if [ $? -eq 0 ]; then + test_print_trc "$VM_TYPE VM is still up" + test_print_trc "time: $SECONDS" + test_print_trc "SSHPASS: $SSHPASS" + test_print_trc "PORT: $PORT" + test_print_trc "$VM_TYPE VM closed" + # must die here since TDVM should be closed and not accessible if test complete all correctly + # else it's due to test die before reaching final close point td_test_close function + die "$VM_TYPE VM test fail, please check test log" + fi +else # [ $GCOV == "on" ] || [ $? -eq 0 ] + test_print_trc "${VM_TYPE}vm_$PORT keep alive for gcov data collection" + test_print_trc "'ssh -p $PORT root@localhost' with PASSWORD '$SSHPASS' to login and get data" +fi + +## PART 5: clean up at test execution ends, kill tdvm_$PORT process if it's still up ## +# VM life-cycles management step 3 +# Kill the tdvm_$PORT process in case above ssh command close not accessible due to network or other issues +if [ $GCOV == "off" ]; then + pgrep "${VM_TYPE}vm_$PORT" + if [ $? -ne 0 ]; then + test_print_trc "$VM_TYPE VM test complete all correctly..." + else + pkill "${VM_TYPE}vm_$PORT" + test_print_wrg "${VM_TYPE}vm_$PORT process is still up, kill it since test expected to end here" + die "$VM_TYPE VM test fail, please check test log" + fi +fi From f76badf4e12facff61804377c97ac7cf79dbccdd Mon Sep 17 00:00:00 2001 From: Hongyu Ning Date: Wed, 27 Sep 2023 15:12:28 +0800 Subject: [PATCH 3/8] [New][guest-test]: add qemu.config template and parse script qemu.config.json template initial commit, parse script added along to match it Signed-off-by: Hongyu Ning --- guest-test/qemu.config.json | 49 ++++++++ guest-test/qemu_get_config.py | 229 ++++++++++++++++++++++++++++++++++ 2 files changed, 278 insertions(+) create mode 100644 guest-test/qemu.config.json create mode 100755 guest-test/qemu_get_config.py diff --git a/guest-test/qemu.config.json b/guest-test/qemu.config.json new file mode 100644 index 00000000..63fafcaa --- /dev/null +++ b/guest-test/qemu.config.json @@ -0,0 +1,49 @@ +{ + "common": { + "kernel_img": "/tdx/home/sdp/tdx/guest_kernel_github/tdx/arch/x86/boot/bzImage.guest-next.0105", + "initrd_img": "/boot/initramfs-xxx-yyy", + "bios_img": "/tdx/home/sdp/tdx/hongyu/OVMF.edk2-stable202211.fd", + "qemu_img": "/tdx/home/sdp/tdx/host_qemu_github/qemu-tdx/build/qemu-system-x86_64.tdx-qemu-2023-3-13-v7.2-kvm-upstream-2023.03.10-v6.2-wa", + "guest_img": "/tdx/home/sdp/tdx/hongyu/emr-bkc-centos-stream-9-coreserver-host-6.2-13.3-43.image", + "guest_img_format": "raw", + "boot_pattern": "*Kernel*on*x86_64*", + "guest_root_passwd": "123456", + "vm_type": "tdx", + "pmu": "off", + "cpus": "4", + "sockets": "1", + "mem": "16", + "cmdline": "accept_memory=lazy", + "debug": "on" + }, + + "vm": { + "cfg_1": "-accel kvm -no-reboot -nographic -vga none -device virtio-net-pci,netdev=mynet0,mac=DE:AD:BE:EF:AB:CD,romfile= ", + "cfg_2": "-chardev stdio,id=mux,mux=on,signal=off -device virtio-serial,romfile= -device virtconsole,chardev=mux ", + "cfg_3": "-serial chardev:mux -monitor chardev:mux -monitor pty -no-hpet -nodefaults ", + "cfg_var_1": "-name process=$VM_TYPEVM_$PORT,debug-threads=on ", + "cfg_var_2": "-cpu host,host-phys-bits,pmu=$PMU ", + "cfg_var_3": "-smp cpus=$VCPU,sockets=$SOCKETS ", + "cfg_var_4": "-m $MEMG ", + "cfg_var_5": "-kernel $KERNEL_IMG ", + "cfg_var_6": "-initrd $INITRD_IMG", + "cfg_var_7": "-netdev user,id=mynet0,hostfwd=tcp::$PORT-:22 ", + "cfg_var_8": "-drive file=$GUEST_IMG,if=virtio,format=$IMG_FORMAT ", + "cfg_var_9": "-append \"root=/dev/vda3 ro console=hvc0 earlyprintk=ttyS0 ignore_loglevel debug earlyprintk l1tf=off initcall_debug log_buf_len=200M nokaslr tsc=reliable efi=debug mce=off efi=debug $CMDLINE\" ", + "cfg_var_10": "-bios $BIOS_IMG " + }, + + "tdx": { + "cfg_1": "-machine q35,kernel_irqchip=split,confidential-guest-support=tdx,memory-backend=ram1 ", + "cfg_var_1": "-object tdx-guest,id=tdx,debug=$DEBUG,sept-ve-disable=on,quote-generation-service=vsock:2:4050 ", + "cfg_var_2": "-object memory-backend-memfd-private,id=ram1,size=$MEMG " + }, + + "tdxio": { + "cfg_1": "-object iommufd,id=iommufd0 ", + "cfg_2": "-device vfio-pci,host=tee_bdf1,id=hostdev2,addr=0x3,x-secure-mode=on ", + "cfg_3": "-device vfio-pci,host=tee_bdf2,id=hostdev3,addr=0x4,x-secure-mode=on ", + "cfg_4": "-device vfio-pci,host=tee_bdf3,id=hostdev4,addr=0x5,x-secure-mode=on ", + "cfg_5": "-device vfio-pci,host=tee_bdf4,id=hostdev5,addr=0x6,x-secure-mode=on " + } +} \ No newline at end of file diff --git a/guest-test/qemu_get_config.py b/guest-test/qemu_get_config.py new file mode 100755 index 00000000..2179ad3a --- /dev/null +++ b/guest-test/qemu_get_config.py @@ -0,0 +1,229 @@ +#!/usr/bin/python3 + +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (c) 2023 Intel Corporation + +# Author: Hongyu Ning +# +# History: 7, Sep., 2023 - Hongyu Ning - creation + + +# @desc This script read qemu config from qemu.config.json +# and pass to qemu runner for vm launching +# @ params source 1: general params exported from tdx.config +# @ params source 2: test scenario config from tdx.test_params.sh + +###################### Lib and Module ###################### +import os +from pathlib import Path +import json +from test_params import * +import argparse + +###################### Variables ###################### + +# read from qemu.config.json format for all raw qemu vm config +cwd = Path(os.getcwd()) +if cwd.stem == "guest-test": + raw_config = Path(f"{os.getcwd()}/qemu.config.json").read_text() +else: + exit(1) + +qemu_config = json.loads(raw_config) + +# pre-config G-list variables' values confirmed by qemu.config.json +kernel_img = qemu_config["common"]["kernel_img"] +initrd_img = qemu_config["common"]["initrd_img"] +bios_img = qemu_config["common"]["bios_img"] +qemu_img = qemu_config["common"]["qemu_img"] +guest_img = qemu_config["common"]["guest_img"] +guest_img_format = qemu_config["common"]["guest_img_format"] +boot_pattern = qemu_config["common"]["boot_pattern"] +guest_root_passwd = qemu_config["common"]["guest_root_passwd"] +port = PORT + +# print above G-list variables to test_launcher.sh to export for global shell scripts access +# NOTICE!! DON'T interrupt before any of the following print to avoid mis-behaviors +print(kernel_img) # shell awk $1 +print(initrd_img) # shell awk $2 +print(bios_img) # shell awk $3 +print(qemu_img) # shell awk $4 +print(guest_img) # shell awk $5 +print(guest_img_format) # shell awk $6 +print(boot_pattern) # shell awk $7 +print(guest_root_passwd) # shell awk $8 +#print(port) # shell awk $9 +# NOTICE!! DON'T interrupt before any of the above print to avoid mis-behaviors + +# end of G-list variables handling + +# pre-config O-list variables' values could be override by test_params.py if passed in +# test_params.py is generated by test_launcher.sh for both qemu_runner.py and test_executor.sh +# O-list variables default value from qemu.config.json +vm_type = qemu_config["common"]["vm_type"] +if PMU != None: + pmu = PMU +else: + pmu = qemu_config["common"]["pmu"] + +if VCPU != None: + cpus = VCPU +else: + cpus = qemu_config["common"]["cpus"] + +if SOCKETS != None: + sockets = SOCKETS +else: + sockets = qemu_config["common"]["sockets"] + +if MEM != None: + mem = MEM +else: + mem = qemu_config["common"]["mem"] + +if CMDLINE != None: + cmdline = CMDLINE +else: + cmdline = qemu_config["common"]["cmdline"] + +if DEBUG != None: + debug = DEBUG +else: + debug = qemu_config["common"]["debug"] + +if TESTCASE != None: + testcase = TESTCASE +else: + print("No TESTCASE info found, can't run any test!") + exit(1) + +# O-list variables override value handling with args passed options, not used in framework, keep it for customization +params_o_list = argparse.ArgumentParser() + +params_o_list.add_argument('--vmtype', type=str, help='vm_type to test, valid value [legacy/tdx/tdxio]') +params_o_list.add_argument('--pmu', type=str, help='vm pmu enable, valid value [on/off]') +params_o_list.add_argument('--archpebs', type=str, help='vm arch-pebs enable, valid value [on/off]') +params_o_list.add_argument('--cachetopo', type=str, help='vm x-l2-cache-topo set, valid value [core/cluster]') +params_o_list.add_argument('--cpus', type=int, help='vm total virtual cpu number, pay attention to equation of cpus & sockets/dies/clusters/cores/threads') +params_o_list.add_argument('--sockets', type=int, help='vm total sockets number') +params_o_list.add_argument('--dies', type=int, help='vm total dies number') +params_o_list.add_argument('--clusters', type=int, help='vm total clusters number') +params_o_list.add_argument('--cores', type=int, help='vm total cores number per socket') +params_o_list.add_argument('--threads', type=int, help='vm total threads number per core') +params_o_list.add_argument('--mem', type=int, help='vm total memory size in GB') +params_o_list.add_argument('--cmdline', type=str, help='vm extra command line options') +params_o_list.add_argument('--debug', type=str, help='tdx vm debug enable, valid value [on/off]') +params_o_list.add_argument('--testcase', type=str, help='testcase to run in vm') + +args = params_o_list.parse_args() + +# NOTICE!! O-list veriables' value will be override if passed through above args option +if args.vmtype != None: + vm_type = args.vmtype +if args.pmu != None: + pmu = args.pmu +if args.cpus != None: + cpus = args.cpus +if args.sockets != None: + sockets = args.sockets +if args.mem != None: + mem = args.mem +if args.cmdline != None: + cmdline = args.cmdline +if args.debug != None: + debug = args.debug +if args.testcase != None: + testcase = args.testcase + +# end of O-list variables handling + +# update all cfg_var_x with G-list variables (default values from qemu.config.json) and O-list variables (could be override by passed in value) +# NOTICE!! in case of any cfg_var_x update in qemu.config.json, need to revise following code accordingly +qemu_config["vm"]["cfg_var_1"] = qemu_config["vm"]["cfg_var_1"].replace("$VM_TYPE", vm_type).replace("$PORT", str(port)) +qemu_config["vm"]["cfg_var_2"] = qemu_config["vm"]["cfg_var_2"].replace("$PMU", pmu) +qemu_config["vm"]["cfg_var_3"] = qemu_config["vm"]["cfg_var_3"].replace("$VCPU", str(cpus)).replace("$SOCKETS", str(sockets)) +qemu_config["vm"]["cfg_var_4"] = qemu_config["vm"]["cfg_var_4"].replace("$MEM", str(mem)) +qemu_config["vm"]["cfg_var_5"] = qemu_config["vm"]["cfg_var_5"].replace("$KERNEL_IMG", kernel_img) +# bypass -initrd config option in case it's not provided +if os.path.isfile(initrd_img): + qemu_config["vm"]["cfg_var_6"] = qemu_config["vm"]["cfg_var_6"].replace("$INITRD_IMG", initrd_img) +else: + qemu_config["vm"]["cfg_var_6"] = "" + +qemu_config["vm"]["cfg_var_7"] = qemu_config["vm"]["cfg_var_7"].replace("$PORT", str(port)) +qemu_config["vm"]["cfg_var_8"] = qemu_config["vm"]["cfg_var_8"].replace("$GUEST_IMG", guest_img).replace("$IMG_FORMAT", guest_img_format) +qemu_config["vm"]["cfg_var_9"] = qemu_config["vm"]["cfg_var_9"].replace("$CMDLINE", cmdline) +# bypass -bios config option in case it's not provided, default seabios to use +if os.path.isfile(bios_img): + qemu_config["vm"]["cfg_var_10"] = qemu_config["vm"]["cfg_var_10"].replace("$BIOS_IMG", bios_img) +else: + qemu_config["vm"]["cfg_var_10"] = "" + +qemu_config["tdx"]["cfg_var_1"] = qemu_config["tdx"]["cfg_var_1"].replace("$DEBUG", debug) +qemu_config["tdx"]["cfg_var_2"] = qemu_config["tdx"]["cfg_var_2"].replace("$MEM", str(mem)) + +# end of all cfg_var_x update handling + +###################### Functions ###################### +def get_sub_keys(d, key): + """ + Recursively get all 2nd-level keys in a dictionary. + """ + if isinstance(d, dict): + for k, v in d.items(): + if isinstance(v, dict): + if k == key: + for k2 in v.keys(): + yield k2 + +def print_sub_keys(l, key): + """ + Recursively get each 2nd-level key. + """ + print("Key %s has sub-keys:" %(key)) + for i in l: + print(i) + +def get_sub_cfgs(l, key, result=""): + """ + Recursively collect all 2nd-level key cfg string. + """ + for i in l: + result += qemu_config[key][i] + return result + +###################### Do Works ###################### +#common_keys = list(get_sub_keys(qemu_config, "common")) +vm_keys = list(get_sub_keys(qemu_config, "vm")) +tdx_keys = list(get_sub_keys(qemu_config, "tdx")) +tdxio_keys = list(get_sub_keys(qemu_config, "tdxio")) + +#print_sub_keys(vm_keys, "vm") +if vm_type == "legacy": + vm_cfg = get_sub_cfgs(vm_keys, "vm") + print("HERE're all the vm configs to launch legacy vm:") + print("qemu config option, part 1:") + print(vm_cfg) + +#print_sub_keys(tdx_keys, "tdx") +if vm_type == "tdx": + vm_cfg = get_sub_cfgs(vm_keys, "vm") + tdx_cfg = get_sub_cfgs(tdx_keys, "tdx") + print("HERE're all the tdx configs to launch tdx vm:") + print("qemu config option, part 1:") + print(vm_cfg) + print("qemu config option, part 2:") + print(tdx_cfg) + +#print_sub_keys(tdxio_keys, "tdxio") +if vm_type == "tdxio": + vm_cfg = get_sub_cfgs(vm_keys, "vm") + tdx_cfg = get_sub_cfgs(tdx_keys, "tdx") + tdxio_cfg = get_sub_cfgs(tdxio_keys, "tdxio") + print("HERE're all the tdx configs to launch tdxio vm:") + print("qemu config option, part 1:") + print(vm_cfg) + print("qemu config option, part 2:") + print(tdx_cfg) + print("qemu config option, part 3:") + print(tdxio_cfg) From 2b8465aa8a7f74e1e3e8d5b949f72df73aa2201b Mon Sep 17 00:00:00 2001 From: Hongyu Ning Date: Wed, 27 Sep 2023 15:14:49 +0800 Subject: [PATCH 4/8] [New][guest-test]: qemu_runner script initial commit qemu_runner script to launch guest vm based on qemu.config and script parameters args override Signed-off-by: Hongyu Ning --- guest-test/guest.qemu_runner.sh | 88 +++++++++++++++++++++++++++++++++ guest-test/qemu_runner.py | 41 +++++++++++++++ 2 files changed, 129 insertions(+) create mode 100755 guest-test/guest.qemu_runner.sh create mode 100644 guest-test/qemu_runner.py diff --git a/guest-test/guest.qemu_runner.sh b/guest-test/guest.qemu_runner.sh new file mode 100755 index 00000000..a218acb8 --- /dev/null +++ b/guest-test/guest.qemu_runner.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (c) 2023 Intel Corporation + +# Author: Hongyu Ning +# +# History: 24, Aug., 2023 - Hongyu Ning - creation + + +# @desc This script boots VM thru $QEMU_IMG (called by qemu_runner.py) +# @ params source 1: general params exported from qemu_get_config.py +# @ params source 2: test scenario config sourced from test_params.py + +###################### Variables ###################### +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" +echo "$SCRIPT_DIR" + +###################### Functions ###################### + +# function to do TDX/TDXIO VM launching basic pre-check +# list all the variables value +tdx_pre_check() { + if [[ ! -f $KERNEL_IMG ]]; then + test_print_wrg "In qemu.config.json file, need to set guest kernel properly" + die "TDX guest kernel does not exist..." + else + test_print_trc "TDX guest kernel to test: $KERNEL_IMG" + fi + + if [[ ! -f $BIOS_IMG ]]; then + test_print_wrg "In qemu.config.json file, need to set ovmf properly" + die "Virtual BIOS does not exist..." + else + test_print_trc "BIOS to test: $BIOS_IMG" + fi + + if [[ ! -f $QEMU_IMG ]]; then + test_print_wrg "In qemu.config.json file, need to set qemu properly" + die "QEMU does not exist..." + else + test_print_trc "QEMU to test: $QEMU_IMG" + fi + + if [[ ! -f $GUEST_IMG ]]; then + test_print_wrg "In qemu.config.json file, need to set guest OS img properly" + die "Guest OS does not exist..." + else + test_print_trc "Guest OS image to test: $GUEST_IMG" + fi + + if [[ $GUEST_IMG_FORMAT = "qcow2" ]]; then + test_print_trc "Guest OS image format: qcow2" + else + test_print_trc "Guest OS image format: raw" + fi + + test_print_trc "Guest OS root password: $SSHPASS" + test_print_trc "TDX guest config: vcpu $VCPU, socket $SOCKETS, memory ${MEM}GB" + test_print_trc "TDX guest extra config: debug $DEBUG, extra commandline: $CMDLINE" + test_print_trc "TDX guest ssh forward port: $PORT" + + TDX_SYSFS_FILE="/sys/module/kvm_intel/parameters/tdx" + if [[ -f "$TDX_SYSFS_FILE" ]]; then + if [ "Y" != "$(cat $TDX_SYSFS_FILE)" ] ;then + die "TDX not enabled as expected, please check" + else + test_print_trc "TDX enabled, try to launch TD VM now......" + fi + else + die "kvm_intel module tdx params does not exist, plase check" + fi +} + +###################### Do Works ###################### +cd "$(dirname "$0")" 2>/dev/null || exit 1 +source ../.env + +# get test scenario config for qemu_runner +source $SCRIPT_DIR/test_params.py + +# do basic pre-check for TDX/TDXIO VM launching +if [[ $VM_TYPE == "tdx" ]] || [[ $VM_TYPE == "tdxio" ]]; then + tdx_pre_check +fi + +# launch VM by qemu via qemu_runner.py +test_print_trc "qemu_runner start to launch $VM_TYPE VM" +python3 $SCRIPT_DIR/qemu_runner.py \ No newline at end of file diff --git a/guest-test/qemu_runner.py b/guest-test/qemu_runner.py new file mode 100644 index 00000000..6164352e --- /dev/null +++ b/guest-test/qemu_runner.py @@ -0,0 +1,41 @@ +#!/usr/bin/python3 + +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (c) 2023 Intel Corporation + +# Author: Hongyu Ning +# +# History: 12, Sep., 2023 - Hongyu Ning - creation + + +# @desc This script get variables and applicable qemu_config +# from qemu_get_config.py and launch VM through QEMU +# @ params source 1: global variables and qemu_config from qemu_get_config.py +# @ params source 2: override variables from test_params.py + +###################### Lib and Module ###################### +import subprocess as sp +from qemu_get_config import * +from test_params import * + +###################### Variables ###################### +# all variables imported from qemu_get_config and test_params + +###################### Functions ###################### +# all work done in qemu_get_config.py + +###################### Do Works ###################### +# launch legacy common vm based on vm_type config +if vm_type == "legacy": + command = '{} {}'.format(qemu_img, vm_cfg) + sp.run(command, shell=True) + +# launch tdx vm based on vm_type config +if vm_type == "tdx": + command = '{} {} {}'.format(qemu_img, vm_cfg, tdx_cfg) + sp.run(command, shell=True) + +# launch tdxio vm based on vm_type config +if vm_type == "tdxio": + command = '{} {} {} {}'.format(qemu_img, vm_cfg, tdx_cfg, tdxio_cfg) + sp.run(command, shell=True) From 0cdc46fb86c8eaa5a8c7071d742ff69283b0c434 Mon Sep 17 00:00:00 2001 From: Hongyu Ning Date: Wed, 27 Sep 2023 15:17:10 +0800 Subject: [PATCH 5/8] [New][guest-test]: test_executor initial commit test_executor includes basic framework to prepare and run tests in guest vm based on $TESTCASE Signed-off-by: Hongyu Ning --- guest-test/guest.test_executor.sh | 111 ++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100755 guest-test/guest.test_executor.sh diff --git a/guest-test/guest.test_executor.sh b/guest-test/guest.test_executor.sh new file mode 100755 index 00000000..4bfdf368 --- /dev/null +++ b/guest-test/guest.test_executor.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (c) 2023 Intel Corporation + +# Author: Hongyu Ning +# +# History: 24, Aug., 2023 - Hongyu Ning - creation + + +# @desc This script prepare and run $TESTCASE in Guest VM + +###################### Variables ###################### +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" +echo "$SCRIPT_DIR" +GUEST_TEST_DIR="/root/guest_test/" + +###################### Functions ###################### + +# function based on sshpass to scp common.sh and $1 test_script.sh to Guest VM +guest_test_prepare() { + rm -rf common.sh + wget https://raw.githubusercontent.com/intel/lkvs/main/common/common.sh + sshpass -e ssh -p "$PORT" -o StrictHostKeyChecking=no root@localhost << EOF + rm -rf $GUEST_TEST_DIR + mkdir $GUEST_TEST_DIR +EOF + sshpass -e scp -P "$PORT" -o StrictHostKeyChecking=no common.sh root@localhost:$GUEST_TEST_DIR + sshpass -e scp -P "$PORT" -o StrictHostKeyChecking=no $1 root@localhost:$GUEST_TEST_DIR + test_print_trc "Guest VM test script prepare complete" +} + +# function based on sshpass to scp $1 source_code_dir and compile $2 test_binary in Guest VM +guest_test_source_code() { + sshpass -e scp -P "$PORT" -o StrictHostKeyChecking=no -r $1 root@localhost:$GUEST_TEST_DIR + sshpass -e ssh -p "$PORT" -o StrictHostKeyChecking=no root@localhost << EOF + source $GUEST_TEST_DIR/common.sh + cd $GUEST_TEST_DIR/$1 + make || die "Failed to compile source code $1" + if [ -f $2 ]; then + chmod a+x $2 + cp $2 $GUEST_TEST_DIR + else + die "Can't find test binary $2" + fi +EOF + test_print_trc "Guest VM test source code and binary prepare complete" +} + +# function based on sshpass to execute $1 test_script.sh and potential $2 script params in Guest VM +guest_test_entry() { + sshpass -e ssh -p "$PORT" -o StrictHostKeyChecking=no root@localhost << EOF + source $GUEST_TEST_DIR/common.sh + cd $GUEST_TEST_DIR + test_print_trc "guest_test_entry args 1: $1" + test_print_trc "guest_test_entry args 2: $2" + ./$1 $2 +EOF +ERR_NUM=$? +if [ $ERR_NUM -eq 0 ] || [ $ERR_NUM -eq 255 ]; then + return 0 +else + return 1 +fi +} + +# function based on sshpass to close VM +guest_test_close() { + sshpass -e ssh -p "$PORT" -o StrictHostKeyChecking=no root@localhost << EOF + source $GUEST_TEST_DIR/common.sh + test_print_trc "guest test complete, close VM now" + systemctl reboot --reboot-argument=now +EOF + test_print_trc "Guest VM closed properly after test" +} + +###################### Do Works ###################### +cd "$(dirname "$0")" 2>/dev/null || exit 1 +source ../.env + +# get test scenario config for test_executor +source $SCRIPT_DIR/test_params.py + +cd $SCRIPT_DIR +# select test_functions by $TEST_SCENARIO +case "$TESTCASE" in + TD_BOOT) + guest_test_prepare tdx_guest_boot_check.sh + guest_test_entry tdx_guest_boot_check.sh "-v $VCPU -s $SOCKETS -m $MEM" || \ + die "Failed on TD_BOOT test tdx_guest_boot_check.sh -v $VCPU -s $SOCKETS -m $MEM" + if [[ $GCOV == "off" ]]; then + guest_test_close + fi + ;; + GUEST_TESTCASE_EXAMPLE) + guest_test_prepare guest_test.sh + guest_test_source_code test_source_code_dir_example test_binary_example + guest_test_entry guest_test.sh "-t $TESTCASE" || \ + die "Failed on GUEST_TESTCASE_EXAMPLE guest_test.sh -t $TESTCASE" + if [[ $GCOV == "off" ]]; then + guest_test_close + fi + ;; + :) + test_print_err "Must specify the test scenario option by [-t]" + usage && exit 1 + ;; + \?) + test_print_err "Input test case option $TESTCASE is not supported" + usage && exit 1 + ;; +esac \ No newline at end of file From 6999d8d1bdcba7590f4bcb7a7c7787e1ccc9d275 Mon Sep 17 00:00:00 2001 From: Hongyu Ning Date: Wed, 27 Sep 2023 15:18:59 +0800 Subject: [PATCH 6/8] [New][guest-test][tdx]: add $TESTCASE TD_BOOT test script and tests please refer to this implmentation to add more guest vm testcases Signed-off-by: Hongyu Ning --- guest-test/tdx/tdx_guest_boot_check.sh | 63 ++++++++++++++++++++++++++ guest-test/tdx/tests | 10 ++++ 2 files changed, 73 insertions(+) create mode 100755 guest-test/tdx/tdx_guest_boot_check.sh create mode 100644 guest-test/tdx/tests diff --git a/guest-test/tdx/tdx_guest_boot_check.sh b/guest-test/tdx/tdx_guest_boot_check.sh new file mode 100755 index 00000000..36e4382d --- /dev/null +++ b/guest-test/tdx/tdx_guest_boot_check.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (c) 2023 Intel Corporation + +# Author: Hongyu Ning +# +# History: 24, Aug., 2023 - Hongyu Ning - creation + + +# @desc This script do basic TD guest booting check in TDX Guest VM + +###################### Variables ###################### +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" +echo "$SCRIPT_DIR" +source common.sh + +while getopts :v:s:m: arg; do + case $arg in + v) + VCPU=$OPTARG + ;; + s) + SOCKETS=$OPTARG + ;; + m) + MEM=$OPTARG + ;; + esac +done + +###################### Do Works ###################### +# check vcpu and socket number +vcpu_td=$(lscpu | grep "CPU(s)" | head -1 | awk '{print $2}') +sockets_td=$(lscpu | grep "Socket(s)" | awk '{print $2}') +test_print_trc "vcpu_td: $vcpu_td" +test_print_trc "sockets_td: $sockets_td" + +if [[ "$vcpu_td" -ne "$VCPU" ]]; then + die "Guest TD VM boot with vcpu: $vcpu_td (expected $VCPU)" +fi + +if [[ "$sockets_td" -ne "$SOCKETS" ]]; then + die "Guest TD VM boot with sockets: $sockets_td (expected $SOCKETS)" +fi + +# check memory size +mem_td=$(grep "MemTotal" /proc/meminfo | awk '$3=="kB" {printf "%.0f\n", $2/(1024*1024)}') +test_print_trc "mem_td: $mem_td" + +# $MEM less than or equal to 4GB need special memory size check +if [[ $MEM -le 4 ]]; then + if [[ $(( $MEM / $mem_td )) -lt 1 ]] || [[ $(( $MEM / $mem_td )) -gt 2 ]]; then + die "Guest TD VM boot with memory: $mem_td GB (expected $MEM GB)" + fi +# $MEM more than 4GB use general memory size check +else + if [[ $(( $MEM / $mem_td )) -ne 1 ]]; then + die "Guest TD VM boot with memory: $mem_td GB (expected $MEM GB)" + fi +fi + +test_print_trc "Guest TD VM boot up successfully with config:" +test_print_trc "vcpu $VCPU, socket $SOCKETS, memory $MEM GB" \ No newline at end of file diff --git a/guest-test/tdx/tests b/guest-test/tdx/tests new file mode 100644 index 00000000..3bd40166 --- /dev/null +++ b/guest-test/tdx/tests @@ -0,0 +1,10 @@ +# case implemented by tdx_guest_boot_check.sh +guest.test_launcher.sh -v 1 -s 1 -m 1 -d on -t tdx -x TD_BOOT -c "accept_memory=lazy" -p off +guest.test_launcher.sh -v 1 -s 1 -m 16 -d on -t tdx -x TD_BOOT -c "accept_memory=lazy" -p off +guest.test_launcher.sh -v 4 -s 1 -m 4 -d on -t tdx -x TD_BOOT -c "accept_memory=lazy" -p off +guest.test_launcher.sh -v 4 -s 2 -m 4 -d on -t tdx -x TD_BOOT -c "accept_memory=lazy" -p off +guest.test_launcher.sh -v 4 -s 2 -m 96 -d on -t tdx -x TD_BOOT -c "accept_memory=lazy" -p off +guest.test_launcher.sh -v 64 -s 8 -m 96 -d on -t tdx -x TD_BOOT -c "accept_memory=lazy" -p off +guest.test_launcher.sh -v 288 -s 1 -m 1 -d on -t tdx -x TD_BOOT -c "accept_memory=lazy" -p off +guest.test_launcher.sh -v 288 -s 8 -m 96 -d on -t tdx -x TD_BOOT -c "accept_memory=lazy" -p off +guest.test_launcher.sh -v 1 -s 1 -m 1 -d off -t tdx -x TD_BOOT -c "accept_memory=lazy" -p off From 70e4293eaa2784feed7921b4eb44bde84e86b097 Mon Sep 17 00:00:00 2001 From: Hongyu Ning Date: Wed, 27 Sep 2023 16:51:41 +0800 Subject: [PATCH 7/8] [guest-test] bug fix: fix path error in test_executor fix path error in test_executor Signed-off-by: Hongyu Ning --- guest-test/guest.test_executor.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guest-test/guest.test_executor.sh b/guest-test/guest.test_executor.sh index 4bfdf368..13369ad9 100755 --- a/guest-test/guest.test_executor.sh +++ b/guest-test/guest.test_executor.sh @@ -84,7 +84,7 @@ cd $SCRIPT_DIR # select test_functions by $TEST_SCENARIO case "$TESTCASE" in TD_BOOT) - guest_test_prepare tdx_guest_boot_check.sh + guest_test_prepare tdx/tdx_guest_boot_check.sh guest_test_entry tdx_guest_boot_check.sh "-v $VCPU -s $SOCKETS -m $MEM" || \ die "Failed on TD_BOOT test tdx_guest_boot_check.sh -v $VCPU -s $SOCKETS -m $MEM" if [[ $GCOV == "off" ]]; then From 2d404c0fa219d2c2f797fc8d4df664af1b62e021 Mon Sep 17 00:00:00 2001 From: Hongyu Ning Date: Thu, 28 Sep 2023 14:23:40 +0800 Subject: [PATCH 8/8] [New][guest-test]: code style improve based on github code check code style improvement based on github code check, plus minor json contents change and qemu_get_config.py print context change Signed-off-by: Hongyu Ning --- guest-test/guest.qemu_runner.sh | 4 +- guest-test/guest.test_executor.sh | 10 +-- guest-test/guest.test_launcher.sh | 105 +++++++++++++------------ guest-test/qemu.config.json | 8 +- guest-test/qemu_get_config.py | 54 ++++++------- guest-test/tdx/tdx_guest_boot_check.sh | 8 +- 6 files changed, 93 insertions(+), 96 deletions(-) diff --git a/guest-test/guest.qemu_runner.sh b/guest-test/guest.qemu_runner.sh index a218acb8..b0bfd854 100755 --- a/guest-test/guest.qemu_runner.sh +++ b/guest-test/guest.qemu_runner.sh @@ -76,7 +76,7 @@ cd "$(dirname "$0")" 2>/dev/null || exit 1 source ../.env # get test scenario config for qemu_runner -source $SCRIPT_DIR/test_params.py +source "$SCRIPT_DIR"/test_params.py # do basic pre-check for TDX/TDXIO VM launching if [[ $VM_TYPE == "tdx" ]] || [[ $VM_TYPE == "tdxio" ]]; then @@ -85,4 +85,4 @@ fi # launch VM by qemu via qemu_runner.py test_print_trc "qemu_runner start to launch $VM_TYPE VM" -python3 $SCRIPT_DIR/qemu_runner.py \ No newline at end of file +python3 "$SCRIPT_DIR"/qemu_runner.py \ No newline at end of file diff --git a/guest-test/guest.test_executor.sh b/guest-test/guest.test_executor.sh index 13369ad9..5627ec2d 100755 --- a/guest-test/guest.test_executor.sh +++ b/guest-test/guest.test_executor.sh @@ -24,14 +24,14 @@ guest_test_prepare() { rm -rf $GUEST_TEST_DIR mkdir $GUEST_TEST_DIR EOF - sshpass -e scp -P "$PORT" -o StrictHostKeyChecking=no common.sh root@localhost:$GUEST_TEST_DIR - sshpass -e scp -P "$PORT" -o StrictHostKeyChecking=no $1 root@localhost:$GUEST_TEST_DIR + sshpass -e scp -P "$PORT" -o StrictHostKeyChecking=no common.sh root@localhost:"$GUEST_TEST_DIR" + sshpass -e scp -P "$PORT" -o StrictHostKeyChecking=no "$1" root@localhost:"$GUEST_TEST_DIR" test_print_trc "Guest VM test script prepare complete" } # function based on sshpass to scp $1 source_code_dir and compile $2 test_binary in Guest VM guest_test_source_code() { - sshpass -e scp -P "$PORT" -o StrictHostKeyChecking=no -r $1 root@localhost:$GUEST_TEST_DIR + sshpass -e scp -P "$PORT" -o StrictHostKeyChecking=no -r "$1" root@localhost:"$GUEST_TEST_DIR" sshpass -e ssh -p "$PORT" -o StrictHostKeyChecking=no root@localhost << EOF source $GUEST_TEST_DIR/common.sh cd $GUEST_TEST_DIR/$1 @@ -78,9 +78,9 @@ cd "$(dirname "$0")" 2>/dev/null || exit 1 source ../.env # get test scenario config for test_executor -source $SCRIPT_DIR/test_params.py +source "$SCRIPT_DIR"/test_params.py -cd $SCRIPT_DIR +cd "$SCRIPT_DIR" || die "fail to switch to $SCRIPT_DIR" # select test_functions by $TEST_SCENARIO case "$TESTCASE" in TD_BOOT) diff --git a/guest-test/guest.test_launcher.sh b/guest-test/guest.test_launcher.sh index ebdda31d..1ab31dbd 100755 --- a/guest-test/guest.test_launcher.sh +++ b/guest-test/guest.test_launcher.sh @@ -25,7 +25,7 @@ echo "$SCRIPT_DIR" GCOV="off" # timeout control in case of TD VM booting hang SECONDS=0 -TIMEOUT=60 +TIMEOUT=300 # EXEC_FLAG=0 shows test_executor being called EXEC_FLAG=1 @@ -67,13 +67,25 @@ NOTE!! args passed here will override params in qemu.confg.json EOF } +guest_kernel_check() { + sshpass -e ssh -p "$PORT" -o StrictHostKeyChecking=no root@localhost << EOF + echo "$VM_TYPE VM guest kernel under test:" + uname -r +EOF +} + +guest_kernel_reboot() { + sshpass -e ssh -p "$PORT" -o StrictHostKeyChecking=no root@localhost << EOF + systemctl reboot --reboot-argument=now +EOF +} + ###################### Do Works ###################### cd "$(dirname "$0")" 2>/dev/null || exit 1 source ../.env ## PART 0: prepare test prerequisites ## -which sshpass > /dev/null -if [ $? -ne 0 ]; then +if [ ! "$(which sshpass)" ]; then dnf install -y sshpass > /dev/null apt install -y sshpass > /dev/null else @@ -85,7 +97,7 @@ fi # generate a unified random port number in one test cycle PORT=$(shuf -i 10010-10900 -n 1) # generate test_params.py based on script args -echo PORT=$PORT > $SCRIPT_DIR/test_params.py +echo PORT="$PORT" > "$SCRIPT_DIR"/test_params.py # 1.1 get test scenario config related params from script args # append following params in above fresh new test_params.py @@ -96,39 +108,42 @@ while getopts :v:s:m:d:t:x:c:p:g:h arg; do case $arg in v) VCPU=$OPTARG - echo VCPU=$VCPU >> $SCRIPT_DIR/test_params.py + echo VCPU="$VCPU" >> "$SCRIPT_DIR"/test_params.py ;; s) SOCKETS=$OPTARG - echo SOCKETS=$SOCKETS >> $SCRIPT_DIR/test_params.py + echo SOCKETS="$SOCKETS" >> "$SCRIPT_DIR"/test_params.py ;; m) MEM=$OPTARG - echo MEM=$MEM >> $SCRIPT_DIR/test_params.py + echo MEM="$MEM" >> "$SCRIPT_DIR"/test_params.py ;; d) DEBUG=$OPTARG - echo DEBUG=\"$DEBUG\" >> $SCRIPT_DIR/test_params.py + echo DEBUG="\"$DEBUG\"" >> "$SCRIPT_DIR"/test_params.py ;; t) VM_TYPE=$OPTARG - echo VM_TYPE=\"$VM_TYPE\" >> $SCRIPT_DIR/test_params.py + echo VM_TYPE="\"$VM_TYPE\"" >> "$SCRIPT_DIR"/test_params.py ;; x) TESTCASE=$OPTARG - echo TESTCASE=\"$TESTCASE\" >> $SCRIPT_DIR/test_params.py + echo TESTCASE="\"$TESTCASE\"" >> "$SCRIPT_DIR"/test_params.py ;; c) CMDLINE=$OPTARG - echo CMDLINE=\"$CMDLINE\" >> $SCRIPT_DIR/test_params.py + echo CMDLINE="\"$CMDLINE\"" >> "$SCRIPT_DIR"/test_params.py ;; p) PMU=$OPTARG - echo PMU=\"$PMU\" >> $SCRIPT_DIR/test_params.py + echo PMU="\"$PMU\"" >> "$SCRIPT_DIR"/test_params.py ;; g) GCOV=$OPTARG - echo GCOV=\"$GCOV\" >> $SCRIPT_DIR/test_params.py + echo GCOV="\"$GCOV\"" >> "$SCRIPT_DIR"/test_params.py + ;; + h) + usage && exit 0 ;; :) test_print_err "Must supply an argument to -$OPTARG." @@ -148,15 +163,15 @@ done # $GUEST_IMG_FORMAT $BOOT_PATTERN $SSHPASS $PORT #global_variable -output=$(python3 $SCRIPT_DIR/qemu_get_config.py) -KERNEL_IMG=$(echo $output | awk '{print $1}') -INITRD_IMG=$(echo $output | awk '{print $2}') -BIOS_IMG=$(echo $output | awk '{print $3}') -QEMU_IMG=$(echo $output | awk '{print $4}') -GUEST_IMG=$(echo $output | awk '{print $5}') -GUEST_IMG_FORMAT=$(echo $output | awk '{print $6}') -BOOT_PATTERN=$(echo $output | awk '{print $7}') -SSHPASS=$(echo $output | awk '{print $8}') +output=$(python3 "$SCRIPT_DIR"/qemu_get_config.py) +KERNEL_IMG=$(echo "$output" | awk '{print $1; exit}') +INITRD_IMG=$(echo "$output" | awk '{print $2; exit}') +BIOS_IMG=$(echo "$output" | awk '{print $3; exit}') +QEMU_IMG=$(echo "$output" | awk '{print $4; exit}') +GUEST_IMG=$(echo "$output" | awk '{print $5; exit}') +GUEST_IMG_FORMAT=$(echo "$output" | awk '{print $6; exit}') +BOOT_PATTERN=$(echo "$output" | awk '{print $7; exit}') +SSHPASS=$(echo "$output" | awk '{print $8; exit}') test_print_trc "KERNEL_IMG $KERNEL_IMG" test_print_trc "INITRD_IMG $INITRD_IMG" @@ -187,7 +202,7 @@ export GCOV # 2.3 check TD VM boot $ERR_STRs, then run corresponding err_handler ($ERR_FLAGs) # 2.4 break while loop if reach $TIMEOUT seconds (in case of TD VM boot hang) -cd $SCRIPT_DIR +cd "$SCRIPT_DIR" || die "fail to switch to $SCRIPT_DIR" rm -rf /root/.ssh/known_hosts while read -r line; do echo "[${VM_TYPE}_vm]: $line" @@ -196,10 +211,7 @@ while read -r line; do if [[ $line == $BOOT_PATTERN ]]; then test_print_trc "VM_TYPE: $VM_TYPE, VCPU: $VCPU, SOCKETS: $SOCKETS, MEM: $MEM, DEBUG: $DEBUG, PMU: $PMU, CMDLINE: $CMDLINE, TESTCASE: $TESTCASE, SECONDS: $SECONDS" EXEC_FLAG=0 - ./guest.test_executor.sh - if [ $? -ne 0 ]; then # break while read loop in case of TD VM test failure - break - fi + ./guest.test_executor.sh || break # break while read loop in case of TD VM test failure # err_handlers string matching elif [[ $line == $ERR_STR1 ]]; then test_print_err "There is $ERR_STR1, test is not fully PASS" @@ -221,7 +233,7 @@ while read -r line; do elif [[ $SECONDS -ge $TIMEOUT ]]; then # break while read loop in case of TD VM boot timeout (no $BOOT_PATTERN found) break fi -done < <(if [ $GCOV == "off" ]; then timeout $TIMEOUT ./guest.qemu_runner.sh; else ./guest.qemu_runner.sh; fi) +done < <(if [ "$GCOV" == "off" ]; then timeout "$TIMEOUT" ./guest.qemu_runner.sh; else ./guest.qemu_runner.sh; fi) ## PART 3: err_handlers error management # unexpected error/bug/warning/call trace handling @@ -251,15 +263,10 @@ fi sleep 3 # VM life-cycles management step 1 -# check if TDVM is still up, non-zero return value indicates TDVM is not accessible +# check if TDVM is still up via guest_kernel_check function, non-zero return value indicates TDVM is not accessible # TDVM not acccessible cases: # a. TDVM is already closed after test # b. TDVM boot up stuck at some point -sshpass -e ssh -p "$PORT" -o StrictHostKeyChecking=no root@localhost << EOF - echo "$VM_TYPE VM guest kernel under test:" - uname -r -EOF - # VM life-cycles management step 2 # non-zero return value of TD VM not accessible handling # time count between 3 and $TIMEOUT is expected case a @@ -268,24 +275,19 @@ EOF # - handling: kill the tdvm_$PORT process since it's stuck # time count less or qual than 3 is case b # - handling: nothing to do, die for TDVM boot early failure, likely qemu config issue -if [ $? -ne 0 ]; then - if [ $SECONDS -gt 3 ] && [ $SECONDS -lt $TIMEOUT ] && [ $EXEC_FLAG -eq 0 ]; then +if ! guest_kernel_check; then + if [ "$SECONDS" -gt 3 ] && [ "$SECONDS" -lt "$TIMEOUT" ] && [ "$EXEC_FLAG" -eq 0 ]; then test_print_trc "$VM_TYPE VM test complete..." - elif [ $SECONDS -ge $TIMEOUT ] && [ $GCOV == "on" ]; then + elif [ "$SECONDS" -ge "$TIMEOUT" ] && [ "$GCOV" == "on" ]; then pkill "${VM_TYPE}vm_$PORT" die "TEST TIMEOUT!!!!!!!!!!!!" - elif [ $GCOV == "off" ]; then + elif [ "$GCOV" == "off" ] && [ "$EXEC_FLAG" -eq 1 ]; then die "$VM_TYPE VM test seems fail at beginning, please check test log" fi -# zero return value shows TDVM is still accessible handling -# handing: no matter why it's still accessible, close it by ssh login -elif [ $GCOV == "off" ]; then - sshpass -e ssh -p "$PORT" -o StrictHostKeyChecking=no root@localhost << EOF - echo "$VM_TYPE VM guest kernel under test:" - uname -r - systemctl reboot --reboot-argument=now -EOF - if [ $? -eq 0 ]; then +# guest_kernel_kernel function zero return value shows TDVM is still accessible handling +# handling: no matter why it's still accessible, close it by guest_kernel_reboot function +elif [ "$GCOV" == "off" ]; then + if ! guest_kernel_reboot; then test_print_trc "$VM_TYPE VM is still up" test_print_trc "time: $SECONDS" test_print_trc "SSHPASS: $SSHPASS" @@ -295,7 +297,7 @@ EOF # else it's due to test die before reaching final close point td_test_close function die "$VM_TYPE VM test fail, please check test log" fi -else # [ $GCOV == "on" ] || [ $? -eq 0 ] +else # [ $GCOV == "on" ] || [ guest_kernel_check return 0 ] test_print_trc "${VM_TYPE}vm_$PORT keep alive for gcov data collection" test_print_trc "'ssh -p $PORT root@localhost' with PASSWORD '$SSHPASS' to login and get data" fi @@ -303,13 +305,12 @@ fi ## PART 5: clean up at test execution ends, kill tdvm_$PORT process if it's still up ## # VM life-cycles management step 3 # Kill the tdvm_$PORT process in case above ssh command close not accessible due to network or other issues -if [ $GCOV == "off" ]; then - pgrep "${VM_TYPE}vm_$PORT" - if [ $? -ne 0 ]; then +if [ "$GCOV" == "off" ]; then + if [ ! "$(pgrep "${VM_TYPE}vm_$PORT")" ]; then test_print_trc "$VM_TYPE VM test complete all correctly..." else pkill "${VM_TYPE}vm_$PORT" test_print_wrg "${VM_TYPE}vm_$PORT process is still up, kill it since test expected to end here" die "$VM_TYPE VM test fail, please check test log" fi -fi +fi \ No newline at end of file diff --git a/guest-test/qemu.config.json b/guest-test/qemu.config.json index 63fafcaa..49243370 100644 --- a/guest-test/qemu.config.json +++ b/guest-test/qemu.config.json @@ -1,10 +1,10 @@ { "common": { - "kernel_img": "/tdx/home/sdp/tdx/guest_kernel_github/tdx/arch/x86/boot/bzImage.guest-next.0105", + "kernel_img": "/boot/vmlinuz-xxx-yyy", "initrd_img": "/boot/initramfs-xxx-yyy", - "bios_img": "/tdx/home/sdp/tdx/hongyu/OVMF.edk2-stable202211.fd", - "qemu_img": "/tdx/home/sdp/tdx/host_qemu_github/qemu-tdx/build/qemu-system-x86_64.tdx-qemu-2023-3-13-v7.2-kvm-upstream-2023.03.10-v6.2-wa", - "guest_img": "/tdx/home/sdp/tdx/hongyu/emr-bkc-centos-stream-9-coreserver-host-6.2-13.3-43.image", + "bios_img": "/path/to/EDKII/OVMF.fd or other virtual BIOS", + "qemu_img": "/path/to/qemu-kvm with proper capabilty of VM test", + "guest_img": "/path/to/prepared/guest_os_image, in qcow2 or raw image format", "guest_img_format": "raw", "boot_pattern": "*Kernel*on*x86_64*", "guest_root_passwd": "123456", diff --git a/guest-test/qemu_get_config.py b/guest-test/qemu_get_config.py index 2179ad3a..17adcf03 100755 --- a/guest-test/qemu_get_config.py +++ b/guest-test/qemu_get_config.py @@ -44,15 +44,7 @@ # print above G-list variables to test_launcher.sh to export for global shell scripts access # NOTICE!! DON'T interrupt before any of the following print to avoid mis-behaviors -print(kernel_img) # shell awk $1 -print(initrd_img) # shell awk $2 -print(bios_img) # shell awk $3 -print(qemu_img) # shell awk $4 -print(guest_img) # shell awk $5 -print(guest_img_format) # shell awk $6 -print(boot_pattern) # shell awk $7 -print(guest_root_passwd) # shell awk $8 -#print(port) # shell awk $9 +print(kernel_img, initrd_img, bios_img, qemu_img, guest_img, guest_img_format, boot_pattern, guest_root_passwd) # shell awk $1-$8 # NOTICE!! DON'T interrupt before any of the above print to avoid mis-behaviors # end of G-list variables handling @@ -61,37 +53,37 @@ # test_params.py is generated by test_launcher.sh for both qemu_runner.py and test_executor.sh # O-list variables default value from qemu.config.json vm_type = qemu_config["common"]["vm_type"] -if PMU != None: +if PMU is not None: pmu = PMU else: pmu = qemu_config["common"]["pmu"] -if VCPU != None: +if VCPU is not None: cpus = VCPU else: cpus = qemu_config["common"]["cpus"] -if SOCKETS != None: +if SOCKETS is not None: sockets = SOCKETS else: sockets = qemu_config["common"]["sockets"] -if MEM != None: +if MEM is not None: mem = MEM else: mem = qemu_config["common"]["mem"] -if CMDLINE != None: +if CMDLINE is not None: cmdline = CMDLINE else: cmdline = qemu_config["common"]["cmdline"] -if DEBUG != None: +if DEBUG is not None: debug = DEBUG else: debug = qemu_config["common"]["debug"] -if TESTCASE != None: +if TESTCASE is not None: testcase = TESTCASE else: print("No TESTCASE info found, can't run any test!") @@ -118,21 +110,21 @@ args = params_o_list.parse_args() # NOTICE!! O-list veriables' value will be override if passed through above args option -if args.vmtype != None: +if args.vmtype is not None: vm_type = args.vmtype -if args.pmu != None: +if args.pmu is not None: pmu = args.pmu -if args.cpus != None: +if args.cpus is not None: cpus = args.cpus -if args.sockets != None: +if args.sockets is not None: sockets = args.sockets -if args.mem != None: +if args.mem is not None: mem = args.mem -if args.cmdline != None: +if args.cmdline is not None: cmdline = args.cmdline -if args.debug != None: +if args.debug is not None: debug = args.debug -if args.testcase != None: +if args.testcase is not None: testcase = args.testcase # end of O-list variables handling @@ -202,7 +194,7 @@ def get_sub_cfgs(l, key, result=""): if vm_type == "legacy": vm_cfg = get_sub_cfgs(vm_keys, "vm") print("HERE're all the vm configs to launch legacy vm:") - print("qemu config option, part 1:") + print("#### qemu config option, part 1 ####") print(vm_cfg) #print_sub_keys(tdx_keys, "tdx") @@ -210,9 +202,9 @@ def get_sub_cfgs(l, key, result=""): vm_cfg = get_sub_cfgs(vm_keys, "vm") tdx_cfg = get_sub_cfgs(tdx_keys, "tdx") print("HERE're all the tdx configs to launch tdx vm:") - print("qemu config option, part 1:") + print("#### qemu config option, part 1 ####") print(vm_cfg) - print("qemu config option, part 2:") + print("#### qemu config option, part 2 ####") print(tdx_cfg) #print_sub_keys(tdxio_keys, "tdxio") @@ -221,9 +213,9 @@ def get_sub_cfgs(l, key, result=""): tdx_cfg = get_sub_cfgs(tdx_keys, "tdx") tdxio_cfg = get_sub_cfgs(tdxio_keys, "tdxio") print("HERE're all the tdx configs to launch tdxio vm:") - print("qemu config option, part 1:") + print("#### qemu config option, part 1 ####") print(vm_cfg) - print("qemu config option, part 2:") + print("#### qemu config option, part 2 ####") print(tdx_cfg) - print("qemu config option, part 3:") - print(tdxio_cfg) + print("#### qemu config option, part 3 ####") + print(tdxio_cfg) \ No newline at end of file diff --git a/guest-test/tdx/tdx_guest_boot_check.sh b/guest-test/tdx/tdx_guest_boot_check.sh index 36e4382d..5830db9b 100755 --- a/guest-test/tdx/tdx_guest_boot_check.sh +++ b/guest-test/tdx/tdx_guest_boot_check.sh @@ -25,6 +25,10 @@ while getopts :v:s:m: arg; do m) MEM=$OPTARG ;; + *) + test_print_err "Must supply an argument to -$OPTARG." + exit 1 + ;; esac done @@ -49,12 +53,12 @@ test_print_trc "mem_td: $mem_td" # $MEM less than or equal to 4GB need special memory size check if [[ $MEM -le 4 ]]; then - if [[ $(( $MEM / $mem_td )) -lt 1 ]] || [[ $(( $MEM / $mem_td )) -gt 2 ]]; then + if [[ $(( MEM / mem_td )) -lt 1 ]] || [[ $(( MEM / mem_td )) -gt 2 ]]; then die "Guest TD VM boot with memory: $mem_td GB (expected $MEM GB)" fi # $MEM more than 4GB use general memory size check else - if [[ $(( $MEM / $mem_td )) -ne 1 ]]; then + if [[ $(( MEM / mem_td )) -ne 1 ]]; then die "Guest TD VM boot with memory: $mem_td GB (expected $MEM GB)" fi fi