Add NUMA node id to device plugin response

pkg/device_plugin/device_plugin.go

@@ -35,6 +35,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 
 	klog "k8s.io/klog/v2"
@@ -62,11 +63,15 @@ var vGpuMap map[string][]NvidiaGpuDevice
 // Key is the Nvidia GPU id and value is the list of associated vGPU ids
 var gpuVgpuMap map[string][]string
 
+// deviceNumaMap is a map of device id to NUMA node id
+var deviceNumaMap map[string]int
+
 var basePath = "/sys/bus/pci/devices"
 var vGpuBasePath = "/sys/bus/mdev/devices"
 var pciIdsFilePath = "/usr/pci.ids"
 var readLink = readLinkFunc
 var readIDFromFile = readIDFromFileFunc
+var readNUMAnodeIDFromFile = readNUMAnodeIDFromFileFunc
 var startDevicePlugin = startDevicePluginFunc
 var readVgpuIDFromFile = readVgpuIDFromFileFunc
 var readGpuIDForVgpu = readGpuIDForVgpuFunc
@@ -91,15 +96,25 @@ func createDevicePlugins() {
 	log.Printf("Device Map %s", deviceMap)
 	log.Println("vGPU Map ", vGpuMap)
 	log.Println("GPU vGPU Map ", gpuVgpuMap)
+	log.Println("Device NUMA Map ", deviceNumaMap)
 
 	//Iterate over deivceMap to create device plugin for each type of GPU on the host
 	for k, v := range deviceMap {
 		devs = nil
 		for _, dev := range v {
-			devs = append(devs, &pluginapi.Device{
+			device := &pluginapi.Device{
 				ID:     dev,
 				Health: pluginapi.Healthy,
-			})
+			}
+			numa, found := deviceNumaMap[dev]
+			if found {
+				device.Topology = &pluginapi.TopologyInfo{
+					Nodes: []*pluginapi.NUMANode{{ID: int64(numa)}},
+				}
+			} else {
+				log.Printf("Error: Could not find NUMA node for device id: %s", dev)
+			}
+			devs = append(devs, device)
 		}
 		deviceName := getDeviceName(k)
 		if deviceName == "" {
@@ -162,6 +177,7 @@ func startVgpuDevicePluginFunc(dp *GenericVGpuDevicePlugin) error {
 func createIommuDeviceMap() {
 	iommuMap = make(map[string][]NvidiaGpuDevice)
 	deviceMap = make(map[string][]string)
+	deviceNumaMap = make(map[string]int)
 	//Walk directory to discover pci devices
 	filepath.Walk(basePath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -206,6 +222,15 @@ func createIommuDeviceMap() {
 					deviceMap[deviceID] = append(deviceMap[deviceID], iommuGroup)
 				}
 				iommuMap[iommuGroup] = append(iommuMap[iommuGroup], NvidiaGpuDevice{info.Name()})
+				numaID, err := readNUMAnodeIDFromFile(basePath, info.Name())
+				if err != nil {
+					log.Println("Could not get numa node id for device ", info.Name())
+					return nil
+				}
+				if numaID != nil {
+					log.Printf("NUMA node for device %s is %d", info.Name(), *numaID)
+					deviceNumaMap[iommuGroup] = *numaID
+				}
 			}
 		}
 		return nil
@@ -257,6 +282,20 @@ func readIDFromFileFunc(basePath string, deviceAddress string, property string)
 	return id, nil
 }
 
+func readNUMAnodeIDFromFileFunc(basePath string, deviceAddress string) (*int, error) {
+	numaContent, err := ioutil.ReadFile(filepath.Join(basePath, deviceAddress, "numa_node"))
+	if err != nil {
+		glog.Errorf("Could not read NUMA node id for device %s, err %s", deviceAddress, err)
+		return nil, err
+	}
+	numaID, err := strconv.Atoi(strings.Trim(string(numaContent), " \n"))
+	if err != nil {
+		glog.Errorf("Could not convert to int NUMA node id for device %s, err %s", deviceAddress, err)
+		return nil, err
+	}
+	return &numaID, nil
+}
+
 // Read a file link
 func readLinkFunc(basePath string, deviceAddress string, link string) (string, error) {
 	path, err := os.Readlink(filepath.Join(basePath, deviceAddress, link))

pkg/device_plugin/device_plugin_test.go

@@ -97,6 +97,17 @@ func getFakeIDFromFileDevicePlugin(basePath string, deviceAddress string, link s
 	return "", errors.New("Incorrect operation")
 }
 
+func getFakeNUMAnodeIDFromDevicePlugin(basePath string, deviceAddress string) (*int, error) {
+	if deviceAddress == deviceAddress1 {
+		fakeNUMANodeID := 0
+		return &fakeNUMANodeID, nil
+	} else if deviceAddress == deviceAddress2 {
+		fakeNUMANodeID := 1
+		return &fakeNUMANodeID, nil
+	}
+	return nil, errors.New("Incorrect operation")
+}
+
 func fakeStartDevicePluginFunc(dp *GenericDevicePlugin) error {
 	if dp.deviceName == deviceName {
 		return errors.New("Incorrect operation")
@@ -186,6 +197,40 @@ var _ = Describe("Device Plugin", func() {
 		})
 	})
 
+	Context("readNUMAnodeIDFromFileFunc() Tests", func() {
+		BeforeEach(func() {
+			workDir, err = ioutil.TempDir("", "kubevirt-test")
+			Expect(err).ToNot(HaveOccurred())
+			os.Mkdir(workDir+"/1", 0755)
+			ioutil.WriteFile(filepath.Join(workDir, deviceAddress1, "numa_node"), []byte("0\n"), 0644)
+		})
+
+		It("Read numa node id with out error", func() {
+			nodeID, err := readNUMAnodeIDFromFileFunc(workDir, deviceAddress1)
+			Expect(err).To(BeNil())
+			Expect(nodeID).ToNot(BeNil())
+			Expect(*nodeID).To(Equal(numaNodeID))
+		})
+
+		It("Read numa node id from a missing location to throw error", func() {
+			os.Remove(filepath.Join(workDir, deviceAddress1, "numa_node"))
+
+			nodeID, err := readNUMAnodeIDFromFileFunc(workDir, deviceAddress1)
+			Expect(err).NotTo(BeNil())
+			var nilNumaNodeID *int
+			Expect(nodeID).To(Equal(nilNumaNodeID))
+		})
+
+		It("Incorrect value of numa node id to throw error", func() {
+			ioutil.WriteFile(filepath.Join(workDir, deviceAddress1, "numa_node"), []byte("incorrect\n"), 0644)
+
+			nodeID, err := readNUMAnodeIDFromFileFunc(workDir, deviceAddress1)
+			Expect(err).NotTo(BeNil())
+			var nilNumaNodeID *int
+			Expect(nodeID).To(Equal(nilNumaNodeID))
+		})
+	})
+
 	Context("readVgpuIDFromFile() Tests", func() {
 		BeforeEach(func() {
 			readVgpuIDFromFile = readVgpuIDFromFileFunc
@@ -272,6 +317,7 @@ var _ = Describe("Device Plugin", func() {
 		It("", func() {
 			readLink = getFakeLinkDevicePlugin
 			readIDFromFile = getFakeIDFromFileDevicePlugin
+			readNUMAnodeIDFromFile = getFakeNUMAnodeIDFromDevicePlugin
 			startDevicePlugin = fakeStartDevicePluginFunc
 			createIommuDeviceMap()
 

pkg/device_plugin/generic_device_plugin_test.go

@@ -50,6 +50,7 @@ var pciAddress1 = "11"
 var pciAddress2 = "22"
 var pciAddress3 = "33"
 var nvVendorID = "10de"
+var numaNodeID = 0
 
 type fakeDevicePluginListAndWatchServer struct {
 	grpc.ServerStream