From d5bbebbd4d4cee28792d49a49517208d8570ab74 Mon Sep 17 00:00:00 2001 From: Bryan Humlie Date: Thu, 12 Dec 2024 22:04:56 +0000 Subject: [PATCH 1/4] Restore IDPF devices for renaming rules This reverts commit a528f221688831c3c48425c7d5f6b38c0a21eb8f. --- src/lib/udev/rules.d/75-gce-network.rules | 3 +++ src/usr/bin/gce-nic-naming | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lib/udev/rules.d/75-gce-network.rules b/src/lib/udev/rules.d/75-gce-network.rules index 785e9e1..f566135 100644 --- a/src/lib/udev/rules.d/75-gce-network.rules +++ b/src/lib/udev/rules.d/75-gce-network.rules @@ -16,3 +16,6 @@ IMPORT{builtin}="net_id" # Rule to rename Mellanox devices SUBSYSTEM=="net", ACTION=="add", DRIVERS=="mlx5_core", PROGRAM="/usr/bin/gce-nic-naming", NAME="%c" + +# Rule to rename RDMA devices +SUBSYSTEM=="net", ACTION=="add", DRIVERS=="idpf", PROGRAM="/usr/bin/gce-nic-naming", NAME="%c" \ No newline at end of file diff --git a/src/usr/bin/gce-nic-naming b/src/usr/bin/gce-nic-naming index 027d1aa..570e4a6 100755 --- a/src/usr/bin/gce-nic-naming +++ b/src/usr/bin/gce-nic-naming @@ -21,8 +21,8 @@ readonly LOG_TAG=${LOG_TAG:-"gce-nic-naming_$$"} declare PCI_BUS_PATH='/sys/bus/pci/devices' declare SYS_PREPEND_PATH='/sys' # 0x15b3:0x101e is the vendor and device ID for Mellanox CX7 - -readonly ETHERNET_DEVICES_VENDORS=('15b3:101e') +# 0x8086:0x145c is the vendor and device ID for Intel IDPF +readonly ETHERNET_DEVICES_VENDORS=('15b3:101e' '8086:145c') # 0x10de is the vendor ID for Nvidia readonly GPU_DEVICES_VENDORS=('10de' '10de') # PCI BUS ID path is in the format of 0000:00:04.0 From 4bfa8e6ba11b4ca0d91a18d65d3a62c52d7f2f5b Mon Sep 17 00:00:00 2001 From: Bryan Humlie Date: Fri, 13 Dec 2024 19:28:12 +0000 Subject: [PATCH 2/4] Update comment to indicate specific Intel IDPF device this affects --- src/usr/bin/gce-nic-naming | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/usr/bin/gce-nic-naming b/src/usr/bin/gce-nic-naming index 570e4a6..2d7e429 100755 --- a/src/usr/bin/gce-nic-naming +++ b/src/usr/bin/gce-nic-naming @@ -21,7 +21,7 @@ readonly LOG_TAG=${LOG_TAG:-"gce-nic-naming_$$"} declare PCI_BUS_PATH='/sys/bus/pci/devices' declare SYS_PREPEND_PATH='/sys' # 0x15b3:0x101e is the vendor and device ID for Mellanox CX7 -# 0x8086:0x145c is the vendor and device ID for Intel IDPF +# 0x8086:0x145c is the vendor and device ID for Intel IDPF VF readonly ETHERNET_DEVICES_VENDORS=('15b3:101e' '8086:145c') # 0x10de is the vendor ID for Nvidia readonly GPU_DEVICES_VENDORS=('10de' '10de') From 13f9fd224954fbd0cf669cce51f731f5ef3804fb Mon Sep 17 00:00:00 2001 From: Bryan Humlie Date: Mon, 16 Dec 2024 22:36:59 +0000 Subject: [PATCH 3/4] Handle cases for idpf RDMA devices. --- src/lib/udev/rules.d/75-gce-network.rules | 3 +- src/usr/bin/gce-nic-naming | 37 +++++++++++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/lib/udev/rules.d/75-gce-network.rules b/src/lib/udev/rules.d/75-gce-network.rules index f566135..15f7909 100644 --- a/src/lib/udev/rules.d/75-gce-network.rules +++ b/src/lib/udev/rules.d/75-gce-network.rules @@ -18,4 +18,5 @@ IMPORT{builtin}="net_id" SUBSYSTEM=="net", ACTION=="add", DRIVERS=="mlx5_core", PROGRAM="/usr/bin/gce-nic-naming", NAME="%c" # Rule to rename RDMA devices -SUBSYSTEM=="net", ACTION=="add", DRIVERS=="idpf", PROGRAM="/usr/bin/gce-nic-naming", NAME="%c" \ No newline at end of file +SUBSYSTEM=="net", ACTION=="add", DRIVERS=="idpf", PROGRAM="/usr/bin/gce-nic-naming infiniband", NAME="%c" +SUBSYSTEM=="auxiliary", ACTION=="add", DRIVERS=="*irdma*", PROGRAM="/usr/bin/gce-nic-naming" diff --git a/src/usr/bin/gce-nic-naming b/src/usr/bin/gce-nic-naming index 2d7e429..615b27e 100755 --- a/src/usr/bin/gce-nic-naming +++ b/src/usr/bin/gce-nic-naming @@ -14,6 +14,10 @@ # # This script is used to generate the network device name for a given device. + +# Input to test if a directory in the PCI tree exists under the device +declare PCI_TEST_ITEM="$1" + # Generate a instance specific tag for logging to journal readonly LOG_TAG=${LOG_TAG:-"gce-nic-naming_$$"} @@ -457,7 +461,14 @@ function generate_name() { if [[ ${accelerator_devices[*]} == "" ]] ; then if [[ " ${ETHERNET_DEVICES_VENDORS[*]} " =~ \ [[:space:]]${eth_device_vendor}[[:space:]] ]]; then - name_builder="rdma${eth_index}" + + if [[ "${SUBSYSTEM}" == "net" ]] && [[ ! -d "${SYS_PREPEND_PATH}${DEVPATH}/device/${PCI_TEST_ITEM}" ]]; then + # If this is a VF device and not an RDMA the race condition is addressed by naming to position based name. + notice "Non RDMA NIC. Setting name to path name" + name_builder=$ID_NET_NAME_PATH + else + name_builder="rdma${eth_index}" + fi else error_and_exit "Device is not for intent based name: "\ "Device vendors: eth:${eth_device_vendor}" @@ -478,8 +489,13 @@ function generate_name() { echo ${name_builder} } + # Give the ability to test the script without running the main logic -if [[ ! $TEST == 'test' ]]; then +if [[ ! $TEST == 'test' ]] && [[ ! "${GCE_NIC_NAMING}" == "disable" ]]; then + + if [[ "$SUBSYSTEM" != "net" ]]; then + notice "Triggered for non Net Device" + fi # Note can not use "" around ls path here or it errors out list_devices ethernet_devices accelerator_devices "$(ls -d "${PCI_BUS_PATH}"/*)" @@ -492,5 +508,20 @@ if [[ ! $TEST == 'test' ]]; then determine_index_ratios fi - generate_name "${DEVPATH}" + generated_name=$(generate_name "${DEVPATH}") + + # handle the 2nd load order + if [[ "$SUBSYSTEM" == "net" ]]; then + echo ${generated_name} + else + dev_path="${SYS_PREPEND_PATH}${DEVPATH}/.." + # Check that device has loaded as a net dev before applying the name. + if [[ -d "$dev_path/net/" ]]; then + current_net_iface="$(ls $dev_path/net/)" + notice "Renaming iface ${current_net_iface} to ${generated_name}" + /usr/sbin/ip link set dev ${current_net_iface} down + /usr/sbin/ip link set dev ${current_net_iface} name ${generated_name} + /usr/sbin/ip link set dev ${generated_name} up + fi + fi fi From eacaf41e919760a8682e40fc9d61af33e3ba6fcf Mon Sep 17 00:00:00 2001 From: Bryan Humlie Date: Wed, 18 Dec 2024 23:37:00 +0000 Subject: [PATCH 4/4] Update test to take in the driver value to determine RDMA test case. --- src/lib/udev/rules.d/75-gce-network.rules | 2 +- src/usr/bin/gce-nic-naming | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/lib/udev/rules.d/75-gce-network.rules b/src/lib/udev/rules.d/75-gce-network.rules index 15f7909..2405046 100644 --- a/src/lib/udev/rules.d/75-gce-network.rules +++ b/src/lib/udev/rules.d/75-gce-network.rules @@ -18,5 +18,5 @@ IMPORT{builtin}="net_id" SUBSYSTEM=="net", ACTION=="add", DRIVERS=="mlx5_core", PROGRAM="/usr/bin/gce-nic-naming", NAME="%c" # Rule to rename RDMA devices -SUBSYSTEM=="net", ACTION=="add", DRIVERS=="idpf", PROGRAM="/usr/bin/gce-nic-naming infiniband", NAME="%c" +SUBSYSTEM=="net", ACTION=="add", DRIVERS=="idpf", PROGRAM="/usr/bin/gce-nic-naming %E{ID_NET_DRIVER}", NAME="%c" SUBSYSTEM=="auxiliary", ACTION=="add", DRIVERS=="*irdma*", PROGRAM="/usr/bin/gce-nic-naming" diff --git a/src/usr/bin/gce-nic-naming b/src/usr/bin/gce-nic-naming index 615b27e..47a54bf 100755 --- a/src/usr/bin/gce-nic-naming +++ b/src/usr/bin/gce-nic-naming @@ -16,7 +16,7 @@ # Input to test if a directory in the PCI tree exists under the device -declare PCI_TEST_ITEM="$1" +declare DEVICE_DRIVER="$1" # Generate a instance specific tag for logging to journal readonly LOG_TAG=${LOG_TAG:-"gce-nic-naming_$$"} @@ -462,10 +462,12 @@ function generate_name() { if [[ " ${ETHERNET_DEVICES_VENDORS[*]} " =~ \ [[:space:]]${eth_device_vendor}[[:space:]] ]]; then - if [[ "${SUBSYSTEM}" == "net" ]] && [[ ! -d "${SYS_PREPEND_PATH}${DEVPATH}/device/${PCI_TEST_ITEM}" ]]; then - # If this is a VF device and not an RDMA the race condition is addressed by naming to position based name. - notice "Non RDMA NIC. Setting name to path name" - name_builder=$ID_NET_NAME_PATH + if [[ "${SUBSYSTEM}" == "net" ]] && [[ ! -d "${SYS_PREPEND_PATH}${DEVPATH}/device/${RDMA_TEST_FOLDER}" ]]; then + # If this is a VF device and not an RDMA we do not want this device + # to claim the eth0 name so give it a PCI path based name instead. + # + notice "Non RDMA VF NIC. Setting name to path name" + name_builder=$ID_NET_NAME_PATH else name_builder="rdma${eth_index}" fi @@ -489,6 +491,11 @@ function generate_name() { echo ${name_builder} } +# Intel VF driver and needs special handling to determine if name should be +# RDMA related +if [[ "${DEVICE_DRIVER}" == "idpf" ]]; then + RDMA_TEST_FOLDER="infiniband" +fi # Give the ability to test the script without running the main logic if [[ ! $TEST == 'test' ]] && [[ ! "${GCE_NIC_NAMING}" == "disable" ]]; then @@ -510,10 +517,10 @@ if [[ ! $TEST == 'test' ]] && [[ ! "${GCE_NIC_NAMING}" == "disable" ]]; then generated_name=$(generate_name "${DEVPATH}") - # handle the 2nd load order if [[ "$SUBSYSTEM" == "net" ]]; then echo ${generated_name} else + # Handle load order where netdev plug happens prior to RDMA plug events dev_path="${SYS_PREPEND_PATH}${DEVPATH}/.." # Check that device has loaded as a net dev before applying the name. if [[ -d "$dev_path/net/" ]]; then