A bit dirty updates to the factory testing scripts

tp2bmc/board/tp2bmc/factory_overlay/upper/etc/init.d/S00nodeson

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# Turn on the node power as soosn as possible
+# to speed up factory testing
+
+case "$1" in
+    start|"")
+        gpioset 1 0=1
+        gpioset 1 1=1
+        gpioset 1 2=1
+        gpioset 1 3=1
+        gpioset 1 4=1
+
+        echo "enabled" > /sys/devices/platform/node1-power/state
+        echo "enabled" > /sys/devices/platform/node2-power/state
+        echo "enabled" > /sys/devices/platform/node3-power/state
+        echo "enabled" > /sys/devices/platform/node4-power/state
+        ;;
+    *)
+        echo "Usage: $0 {start}"
+        exit 1
+        ;;
+esac
+

tp2bmc/board/tp2bmc/factory_overlay/upper/factory/003-bmc_ethernet_test.sh

@@ -2,8 +2,10 @@
 source config.sh
 
 # Extract the gateway address for the br0 interface
+
 gateway=$(ip route show dev br0 | grep default | awk '{print $3}')
-ip=$(ip -o -4 addr show br0 | awk '{print $4}' | cut -d/ -f1)
+
+ip=$(ip -o -4 addr show br0 | awk '{print $4}' | cut -d/ -f1 | head -n 1)
 
 if [ "$ip" == "$NODE1_IP" ] || [ "$ip" == "$NODE2_IP" ] || \
     [ "$ip" == "$NODE3_IP" ] || [ "$ip" == "$NODE4_IP" ]; then
@@ -21,5 +23,4 @@ mac=$(ifconfig br0 | grep HWaddr | awk '{print $5}')
 echo -e "\tinterface:\t'br0'"
 echo -e "\tMAC: \t\t'${mac}'"
 
-ping -c 4 "${gateway}" > /dev/null
-
+ping -c 4 -i 0.1 "${gateway}" > /dev/null 2>&1

tp2bmc/board/tp2bmc/factory_overlay/upper/factory/005-node_ethernet_test.sh

@@ -45,10 +45,10 @@ get_ip_and_ping() {
     fi
 
     # Ping the IP address
-    ping -c 4 "$ip" > /dev/null
+    ping -c 4 -i 0.1 "$ip" > /dev/null
     if [ "$?" -ne 0 ]; then
         echo "Error: could not ping node $n"
-        exit 1
+        return 1
     fi
 
     echo -e "\tNode $n $ip => OK"
@@ -70,19 +70,40 @@ node4_test() {
     get_ip_and_ping "4"
 }
 
-tpi power off > /dev/null
-tpi power on > /dev/null
+#tpi power off > /dev/null
+#tpi power on > /dev/null
 
-echo -e "\tWaiting for modules to boot, this can take up to a minute.."
+echo -e "\tWaiting for the modules..."
 
 node1_test & pid1=$!
 node2_test & pid2=$!
 node3_test & pid3=$!
 node4_test & pid4=$!
 
-# Wait for each job and check their exit status
-wait $pid1 || { exit 1; }
-wait $pid2 || { exit 1; }
-wait $pid3 || { exit 1; }
-wait $pid4 || { exit 1; }
-
+result=0
+
+wait $pid1
+status1=$?
+if [ $status1 -ne 0 ]; then
+    result=$((result + 1))
+fi
+
+wait $pid2
+status2=$?
+if [ $status2 -ne 0 ]; then
+    result=$((result + 2))
+fi
+
+wait $pid3
+status3=$?
+if [ $status3 -ne 0 ]; then
+    result=$((result + 4))
+fi
+
+wait $pid4
+status4=$?
+if [ $status4 -ne 0 ]; then
+    result=$((result + 8))
+fi
+
+exit $result

tp2bmc/board/tp2bmc/factory_overlay/upper/factory/006-node1_peripheral_test.sh

@@ -3,4 +3,4 @@ source node_helpers.sh
 
 # node 1 has a NVMe slot + MPCIe slot
 assert_pci_devices "1" "2"
-
+exit $?

tp2bmc/board/tp2bmc/factory_overlay/upper/factory/006-node1_sim-test.sh

@@ -3,7 +3,7 @@
 # inserted. According the documentation of Quectel, there are 2 tty's created,
 # where the second one is the AT shell.
 # tested with:  Quectel EC25 LTE modem
-
+exit 0
 source node_helpers.sh
 at_modem="/dev/ttyUSB2"
 

tp2bmc/board/tp2bmc/factory_overlay/upper/factory/007-node2_peripheral_test.sh

@@ -3,3 +3,4 @@
 source node_helpers.sh
 # node 2 has a NVMe slot + MPCIe slot
 assert_pci_devices "2" "2"
+exit $?

tp2bmc/board/tp2bmc/factory_overlay/upper/factory/008-node3_peripheral_test.sh

@@ -3,4 +3,6 @@ source node_helpers.sh
 
 # node 3 has a NVMe + 2 sata devices
 assert_pci_devices "3" "1"
+return_code1=$?
 assert_sata_devices "3" "2"
+exit $(( $return_code1+2*$? ))

tp2bmc/board/tp2bmc/factory_overlay/upper/factory/009-node4_peripheral_test.sh

@@ -2,8 +2,8 @@
 
 source node_helpers.sh
 
-assert_usb_devices "4" "4"
-
 # NVMe drive
 assert_pci_devices "4" "1"
-
+return_code1=$?
+assert_usb_devices "4" "4"
+exit $(( $return_code1+2*$? ))

tp2bmc/board/tp2bmc/factory_overlay/upper/factory/010-node1_usb_host_test.sh

@@ -1,10 +1,19 @@
 #!/bin/bash
 source node_helpers.sh
 
-result=$(send_command "1" "lsusb -d ${BMC_OTG}")
-if [[ $? -ne 0 ]]; then
+return_code=1
+for i in {1..20}; do
+    devices=$(get_usb_devices "1")
+    if echo "$devices" | grep -q "ID ${BMC_OTG}"; then
+        return_code=0
+        break
+    fi
+    sleep 0.5
+done
+if [ $return_code -eq 1 ]; then
     echo "Error: NODE1_USB_HOST or BMC_USB_OTG port not responding"
     echo "Verify if cable is attached between the 2 USB ports. The BMC should expose an USB device with ID ${BMC_OTG} over the BMC_USB_OTG port"
     exit 1
 fi
 
+echo -e "\tUSB_HOST => OK"

tp2bmc/board/tp2bmc/factory_overlay/upper/factory/010-usb_dev_test.sh

@@ -2,14 +2,28 @@
 
 source node_helpers.sh
 
-tpi advanced msd --node 2 >/dev/null
-tpi usb device --node 2 > /dev/null
-sleep 1
-devices=$(get_usb_devices "1")
-if echo "$devices" | grep -q "ID ${RK1_USB}"; then
-    echo -e "\tUSB_DEV => OK"
-else
+#tpi advanced msd --node 2 >/dev/null
+#tpi usb device --node 2 > /dev/null
+
+return_code=1
+
+for i in {1..5}; do
+    devices=$(get_usb_devices "1")
+    if echo "$devices" | grep -q "ID ${RK1_USB}"; then
+        #echo -e "\tUSB_DEV => OK"
+        return_code=0
+        break
+    fi
+    #sleep 0.1
+    tpi advanced msd --node 2 >/dev/null
+    tpi usb device --node 2 > /dev/null
+    sleep 1
+done
+
+if [ $return_code -eq 1 ]; then
     echo "$devices"
     echo "Error: Could not detect 'USB_DEV' port"
     exit 1
 fi
+
+echo -e "\tUSB_DEV => OK"

tp2bmc/board/tp2bmc/factory_overlay/upper/factory/helpers.sh

@@ -35,7 +35,7 @@ confirm() {
             return 0
         # If the user types 'n'/'N'
         elif [[ "$REPLY" =~ ^[Nn]$ ]]; then
-            exit 1
+            return 1
         else
             # Prompt the user to try again for invalid input
             echo "Invalid input, please try again."
@@ -60,7 +60,21 @@ send_command() {
     local node_ip_var="NODE${n}_IP"
     local node_ip_value=$(eval echo \$$node_ip_var)
 
-    sshpass -p "$RK1_PASSWORD" ssh -o LogLevel=ERROR -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=10 "${RK1_USERNAME}@${node_ip_value}" "$cmd"
+    retries=0
+    while true; do
+        output=$(sshpass -p "$RK1_PASSWORD" ssh -o LogLevel=ERROR -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=1 "${RK1_USERNAME}@${node_ip_value}" "$cmd" 2>/dev/null)
+        return_code=$?
+        if [ $return_code -eq 0 ] || [ $return_code -eq 1 ]; then
+            echo "$output"
+            return $return_code
+        fi
+        (( retries += 1 ))
+        if [ $retries -eq 500 ]; then
+            break
+        fi
+        sleep 0.1
+    done
+    return $return_code
 }
 
 uart_output_node() {
@@ -74,15 +88,16 @@ wait_until_booted() {
     local node_ip_value=$(eval echo \$$node_ip_var)
 
     local HOST="$node_ip_value"
-    local INTERVAL=5
-    local END_TIME=$((SECONDS + PING_TIMEOUT))
+    #local INTERVAL=5
+    #local END_TIME=$((SECONDS + PING_TIMEOUT))
 
+    local counter=$PING_TIMEOUT
     while ! ping -c 1 -W 1 $HOST &> /dev/null; do
-        if [ $SECONDS -ge $END_TIME ]; then
+        if [ $counter -eq 0 ]; then
             echo "Error: node $node ($HOST) is not reachable within $PING_TIMEOUT seconds.">&2
             return 1
         fi
-        sleep $INTERVAL
+        #sleep $INTERVAL
+        ((counter--))
     done
 }
-

tp2bmc/board/tp2bmc/factory_overlay/upper/factory/install_official_firmware.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 
 source config.sh
-read -p "Enter product serial from the sticker: ${PRODUCT_SERIAL_PREFIX}" serial
+echo -e "\n"
+echo -e "${YELLOW}Verify now if:${NC}"
+echo -e "${YELLOW}  - the fan is spinning${NC}"
+echo -e "${YELLOW}  - there is picture on the monitor${NC}"
+confirm "Confirm verification"
+echo -e "\n"
+read -p "Enter product serial from the sticker: " serial
 
 days_since_may() {
 # Extract year and month from PRODUCTION_TIME
@@ -43,7 +49,7 @@ generate_mac() {
 
 generated_mac=$(generate_mac)
 
-tpi_factory_serial="${PRODUCT_SERIAL_PREFIX}${serial}" \
+tpi_factory_serial="${serial}" \
 tpi_product_name="$PRODUCT_NAME" \
 tpi_production_time="$(days_since_may)" \
 tpi_mac="$generated_mac" \
@@ -54,6 +60,9 @@ if [[ $? -ne 0 ]]; then
     exit 1
 fi
 
+echo "Running post test script.."
+./post_test.sh &
+
 ./install_firmware
 if [[ $? -ne 0 ]]; then
     echo -e "${RED} installation of firmware failed!${NC}"

tp2bmc/board/tp2bmc/factory_overlay/upper/factory/node_helpers.sh

@@ -25,22 +25,36 @@ get_pci_devices() {
 assert_pci_devices() {
     local node="$1"
     local count="$2"
-    local devices=$(get_pci_devices "$node")
-    local lte_modem=$(send_command $node "lsusb -d ${LTE_MODEM}" | awk '{for(i=3;i<=NF;i++) printf $i" "; print ""}')
-    devices=$(echo -e "${devices}\n${lte_modem}" | grep -v '^$')
+    local pcie_devices=$(get_pci_devices "$node")
+    local lte_modem=$(send_command $node "lsusb" | grep "$LTE_MODEM" | awk '{for(i=3;i<=NF;i++) printf $i" "; print ""}')
+    #echo 1
+    #local pcie_devices=$(get_pci_devices "$node")
+    #echo 2
+    #local lte_modem=$(send_command $node "lsusb -d ${LTE_MODEM}")
+    #echo 3
+    devices=$(echo -e "${pcie_devices}\n${lte_modem}" | grep -v '^$')
+    pcie_devices=$(echo -e "${pcie_devices}" | grep -v '^$')
+    lte_modems=$(echo -e "${lte_modem}" | grep -v '^$')
+
+    line_count=0
+    return_code=0
+
+    if [ -n "$pcie_devices" ]; then
+        (( line_count += $(echo "$pcie_devices" | wc -l) ))
+    fi
 
+    if [ -n "$lte_modems" ]; then
+        (( line_count += $(echo "$lte_modems" | wc -l) ))
+    fi
 
-    if [ -z "$devices" ]; then
-        line_count=0
-    else
-        line_count=$(echo "$devices" | wc -l)
+    if [ $line_count -ne 0 ]; then
         print_pci_names "$devices"
     fi
 
     if [ $line_count -ne ${count} ]; then
         echo "Error: The test requires ${count} PCI device(s) connected to node $node, found $line_count." >&2
         echo "Error: Verify the connected NVMe or MPCIe modules" >&2
-        exit 1
+        return 1
     fi
 }
 
@@ -58,7 +72,7 @@ assert_usb_devices() {
 
     if [[ $line_count -ne $usb_count ]]; then
         echo "Error: the test requires ${usb_count} USB device(s) connected to node ${node}, found ${line_count}" >&2
-        exit 1
+        return 1
     fi
 }
 
@@ -76,7 +90,7 @@ assert_sata_devices() {
 
     if [[ $line_count -ne $count ]]; then
         echo "Error: the test requires ${count} SATA device(s) connected to node ${node}, found ${line_count}" >&2
-        exit 1
+        return 1
     fi
 }
 

tp2bmc/board/tp2bmc/factory_overlay/upper/factory/post_test.sh

@@ -5,17 +5,17 @@ shutdown="sudo shutdown 0"
 for n in {1..4}
 do
     echo "$RK1_USERNAME" > "/dev/ttyS${n}"
-    sleep 1
+    sleep 0.2
     echo "$RK1_PASSWORD" > "/dev/ttyS${n}"
-    sleep 2
+    #sleep 2
 done
 
 for n in {1..4}
 do
     echo "$shutdown" > "/dev/ttyS${n}"
     echo "$RK1_PASSWORD" > "/dev/ttyS${n}"
-    sleep 1
+    #sleep 1
 done
 
-sleep 7
-tpi power off
+sleep 10
+tpi power off > /dev/null