Merge pull request google#3516 from madeelibm/s390x-topology

Add s390/s390x CPU topology

info/v1/machine.go

@@ -53,6 +53,8 @@ type Core struct {
 	Caches       []Cache `json:"caches"`
 	UncoreCaches []Cache `json:"uncore_caches"`
 	SocketID     int     `json:"socket_id"`
+	BookID       string  `json:"book_id,omitempty"`
+	DrawerID     string  `json:"drawer_id,omitempty"`
 }
 
 type Cache struct {

machine/machine.go

@@ -21,8 +21,6 @@ import (
 	"path"
 	"regexp"
 
-	// s390/s390x changes
-	"runtime"
 	"strconv"
 	"strings"
 
@@ -223,10 +221,6 @@ func GetMachineSwapCapacity() (uint64, error) {
 
 // GetTopology returns CPU topology reading information from sysfs
 func GetTopology(sysFs sysfs.SysFs) ([]info.Node, int, error) {
-	// s390/s390x changes
-	if isSystemZ() {
-		return nil, getNumCores(), nil
-	}
 	return sysinfo.GetNodesInfo(sysFs)
 }
 
@@ -290,15 +284,3 @@ func isRiscv64() bool {
 func isMips64() bool {
 	return strings.Contains(machineArch, "mips64")
 }
-
-// s390/s390x changes
-func getNumCores() int {
-	maxProcs := runtime.GOMAXPROCS(0)
-	numCPU := runtime.NumCPU()
-
-	if maxProcs < numCPU {
-		return maxProcs
-	}
-
-	return numCPU
-}

machine/topology_test.go

@@ -19,6 +19,7 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"runtime"
 	"sort"
 	"testing"
 
@@ -426,11 +427,113 @@ func TestTopologyWithNodesWithoutCPU(t *testing.T) {
 }
 
 func TestTopologyOnSystemZ(t *testing.T) {
-	machineArch = "s390" // overwrite package variable
-	nodes, cores, err := GetTopology(&fakesysfs.FakeSysFs{})
-	assert.Nil(t, err)
-	assert.Nil(t, nodes)
-	assert.NotNil(t, cores)
+	if runtime.GOARCH != "s390x" {
+		t.Skip("skipping TestTopologyOnSystemZ due to wrong architecture")
+	} else {
+		machineArch = "s390" // overwrite package variable
+		sysFs := &fakesysfs.FakeSysFs{}
+
+		c := sysfs.CacheInfo{
+			Id:    0,
+			Size:  128 * 1024,
+			Type:  "Data",
+			Level: 0,
+			Cpus:  2,
+		}
+		sysFs.SetCacheInfo(c)
+
+		nodesPaths := []string{}
+		sysFs.SetNodesPaths(nodesPaths, nil)
+
+		cpusPaths := map[string][]string{
+			"/sys/devices/system/cpu": {
+				"/sys/devices/system/cpu/cpu0",
+				"/sys/devices/system/cpu/cpu1",
+				"/sys/devices/system/cpu/cpu2",
+				"/sys/devices/system/cpu/cpu3",
+			},
+		}
+		sysFs.SetCPUsPaths(cpusPaths, nil)
+
+		coreThread := map[string]string{
+			"/sys/devices/system/cpu/cpu0": "0",
+			"/sys/devices/system/cpu/cpu1": "1",
+			"/sys/devices/system/cpu/cpu2": "0",
+			"/sys/devices/system/cpu/cpu3": "1",
+		}
+		sysFs.SetCoreThreads(coreThread, nil)
+
+		physicalPackageIDs := map[string]string{
+			"/sys/devices/system/cpu/cpu0": "0",
+			"/sys/devices/system/cpu/cpu1": "1",
+			"/sys/devices/system/cpu/cpu2": "0",
+			"/sys/devices/system/cpu/cpu3": "1",
+		}
+		sysFs.SetPhysicalPackageIDs(physicalPackageIDs, nil)
+
+		bookIDs := map[string]string{
+			"/sys/devices/system/cpu/cpu0": "1",
+			"/sys/devices/system/cpu/cpu1": "1",
+			"/sys/devices/system/cpu/cpu2": "1",
+			"/sys/devices/system/cpu/cpu3": "1",
+		}
+		sysFs.SetBookIDs(bookIDs, nil)
+
+		drawerIDs := map[string]string{
+			"/sys/devices/system/cpu/cpu0": "0",
+			"/sys/devices/system/cpu/cpu1": "0",
+			"/sys/devices/system/cpu/cpu2": "0",
+			"/sys/devices/system/cpu/cpu3": "0",
+		}
+		sysFs.SetDrawerIDs(drawerIDs, nil)
+
+		topology, numCores, err := GetTopology(sysFs)
+		assert.Nil(t, err)
+		assert.Equal(t, 2, len(topology))
+		assert.Equal(t, 4, numCores)
+
+		topologyJSON1, err := json.Marshal(topology[0])
+		assert.Nil(t, err)
+		topologyJSON2, err := json.Marshal(topology[1])
+		assert.Nil(t, err)
+
+		expectedTopology1 := `{"node_id":0,"memory":0,"hugepages":null,"distances":null,"cores":[{"core_id":0,"thread_ids":[0,2],"caches":[{"id":0, "size":131072,"type":"Data","level":0}], "socket_id": 0, "book_id":"1", "drawer_id":"0", "uncore_caches":null}],"caches":null}`
+		expectedTopology2 := `
+		{
+			"node_id":1,
+			"memory":0,
+			"hugepages":null,
+            "distances": null,
+			"cores":[
+				{
+					"core_id":1,
+					"thread_ids":[
+					1,
+					3
+					],
+					"caches":[
+					{
+						"id": 0,
+						"size":131072,
+						"type":"Data",
+						"level":0
+					}
+					],
+					"socket_id": 1,
+					"book_id": "1",
+					"drawer_id": "0",
+					"uncore_caches": null
+				}
+			],
+			"caches":null
+		}`
+
+		json1 := string(topologyJSON1)
+		json2 := string(topologyJSON2)
+
+		assert.JSONEq(t, expectedTopology1, json1)
+		assert.JSONEq(t, expectedTopology2, json2)
+	}
 }
 
 func TestMemoryInfo(t *testing.T) {

utils/sysfs/fakesysfs/fake.go

@@ -67,6 +67,12 @@ type FakeSysFs struct {
 	physicalPackageIDs   map[string]string
 	physicalPackageIDErr map[string]error
 
+	bookIDs   map[string]string
+	bookIDErr map[string]error
+
+	drawerIDs   map[string]string
+	drawerIDErr map[string]error
+
 	memTotal string
 	memErr   error
 
@@ -98,6 +104,14 @@ func (fs *FakeSysFs) GetCPUPhysicalPackageID(cpuPath string) (string, error) {
 	return fs.physicalPackageIDs[cpuPath], fs.physicalPackageIDErr[cpuPath]
 }
 
+func (fs *FakeSysFs) GetBookID(coreIDPath string) (string, error) {
+	return fs.bookIDs[coreIDPath], fs.bookIDErr[coreIDPath]
+}
+
+func (fs *FakeSysFs) GetDrawerID(coreIDPath string) (string, error) {
+	return fs.drawerIDs[coreIDPath], fs.drawerIDErr[coreIDPath]
+}
+
 func (fs *FakeSysFs) GetMemInfo(nodePath string) (string, error) {
 	return fs.memTotal, fs.memErr
 }
@@ -185,6 +199,16 @@ func (fs *FakeSysFs) SetPhysicalPackageIDs(physicalPackageIDs map[string]string,
 	fs.physicalPackageIDErr = physicalPackageIDErrors
 }
 
+func (fs *FakeSysFs) SetBookIDs(bookIDs map[string]string, bookIDErrors map[string]error) {
+	fs.bookIDs = bookIDs
+	fs.bookIDErr = bookIDErrors
+}
+
+func (fs *FakeSysFs) SetDrawerIDs(drawerIDs map[string]string, drawerIDErrors map[string]error) {
+	fs.drawerIDs = drawerIDs
+	fs.drawerIDErr = drawerIDErrors
+}
+
 func (fs *FakeSysFs) SetMemory(memTotal string, err error) {
 	fs.memTotal = memTotal
 	fs.memErr = err

utils/sysfs/sysfs.go

@@ -21,6 +21,7 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 
@@ -50,6 +51,8 @@ const (
 
 	coreIDFilePath    = "/" + sysFsCPUTopology + "/core_id"
 	packageIDFilePath = "/" + sysFsCPUTopology + "/physical_package_id"
+	bookIDFilePath    = "/" + sysFsCPUTopology + "/book_id"
+	drawerIDFilePath  = "/" + sysFsCPUTopology + "/drawer_id"
 
 	// memory size calculations
 
@@ -87,6 +90,10 @@ type SysFs interface {
 	GetCoreID(coreIDFilePath string) (string, error)
 	// Get physical package id for specified CPU
 	GetCPUPhysicalPackageID(cpuPath string) (string, error)
+	// Get book id for specified CPU
+	GetBookID(cpuPath string) (string, error)
+	// Get drawer id for specified CPU
+	GetDrawerID(cpuPath string) (string, error)
 	// Get total memory for specified NUMA node
 	GetMemInfo(nodeDir string) (string, error)
 	// Get hugepages from specified directory
@@ -164,6 +171,24 @@ func (fs *realSysFs) GetCPUPhysicalPackageID(cpuPath string) (string, error) {
 	return strings.TrimSpace(string(packageID)), err
 }
 
+func (fs *realSysFs) GetBookID(cpuPath string) (string, error) {
+	bookIDFilePath := fmt.Sprintf("%s%s", cpuPath, bookIDFilePath)
+	bookID, err := os.ReadFile(bookIDFilePath)
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(bookID)), nil
+}
+
+func (fs *realSysFs) GetDrawerID(cpuPath string) (string, error) {
+	drawerIDFilePath := fmt.Sprintf("%s%s", cpuPath, drawerIDFilePath)
+	drawerID, err := os.ReadFile(drawerIDFilePath)
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(drawerID)), nil
+}
+
 func (fs *realSysFs) GetMemInfo(nodePath string) (string, error) {
 	meminfoPath := fmt.Sprintf("%s/%s", nodePath, meminfoFile)
 	meminfo, err := os.ReadFile(meminfoPath)
@@ -365,22 +390,24 @@ func getCPUCount(cache string) (count int, err error) {
 
 func (fs *realSysFs) GetCacheInfo(cpu int, name string) (CacheInfo, error) {
 	cachePath := fmt.Sprintf("%s%d/cache/%s", cacheDir, cpu, name)
-	out, err := os.ReadFile(path.Join(cachePath, "/id"))
-	if err != nil {
-		return CacheInfo{}, err
-	}
 	var id int
-	n, err := fmt.Sscanf(string(out), "%d", &id)
-	if err != nil || n != 1 {
-		return CacheInfo{}, err
+	if runtime.GOARCH != "s390x" {
+		out, err := os.ReadFile(path.Join(cachePath, "/id"))
+		if err != nil {
+			return CacheInfo{}, err
+		}
+		n, err := fmt.Sscanf(string(out), "%d", &id)
+		if err != nil || n != 1 {
+			return CacheInfo{}, err
+		}
 	}
 
-	out, err = os.ReadFile(path.Join(cachePath, "/size"))
+	out, err := os.ReadFile(path.Join(cachePath, "/size"))
 	if err != nil {
 		return CacheInfo{}, err
 	}
 	var size uint64
-	n, err = fmt.Sscanf(string(out), "%dK", &size)
+	n, err := fmt.Sscanf(string(out), "%dK", &size)
 	if err != nil || n != 1 {
 		return CacheInfo{}, err
 	}

utils/sysinfo/sysinfo.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"os"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 
@@ -464,12 +465,35 @@ func getCoresInfo(sysFs sysfs.SysFs, cpuDirs []string) ([]info.Core, error) {
 			return nil, err
 		}
 
+		var bookID, drawerID string
+		// s390/s390x additional cpu topology levels
+		if runtime.GOARCH == "s390x" {
+			bookID, err = sysFs.GetBookID(cpuDir)
+			if os.IsNotExist(err) {
+				klog.Warningf("Cannot read book id for %s, book_id file does not exist, err: %s", cpuDir, err)
+				continue
+			} else if err != nil {
+				return nil, err
+			}
+			drawerID, err = sysFs.GetDrawerID(cpuDir)
+			if os.IsNotExist(err) {
+				klog.Warningf("Cannot read drawer id for %s, drawer_id file does not exist, err: %s", cpuDir, err)
+				continue
+			} else if err != nil {
+				return nil, err
+			}
+		}
+
 		coreIDx := -1
 		for id, core := range cores {
 			if core.Id == physicalID && core.SocketID == physicalPackageID {
-				coreIDx = id
+				// For s390x, we need to check the BookID and DrawerID match as well.
+				if runtime.GOARCH != "s390x" || (core.BookID == bookID && core.DrawerID == drawerID) {
+					coreIDx = id
+				}
 			}
 		}
+
 		if coreIDx == -1 {
 			cores = append(cores, info.Core{})
 			coreIDx = len(cores) - 1
@@ -478,6 +502,8 @@ func getCoresInfo(sysFs sysfs.SysFs, cpuDirs []string) ([]info.Core, error) {
 
 		desiredCore.Id = physicalID
 		desiredCore.SocketID = physicalPackageID
+		desiredCore.BookID = bookID
+		desiredCore.DrawerID = drawerID
 
 		if len(desiredCore.Threads) == 0 {
 			desiredCore.Threads = []int{cpuID}