From aff2c31671b7b61338917fd3c587c18a0386e3dc Mon Sep 17 00:00:00 2001 From: Brice Goglin Date: Mon, 12 Jun 2023 11:08:05 +0200 Subject: [PATCH 1/4] levelzero: only get memory info from sysman Now that zesInit() is mandatory, don't bother falling back to the core API, Sysman shouldn't fail. Signed-off-by: Brice Goglin --- hwloc/topology-levelzero.c | 97 +------------------ .../include/levelzero/level_zero/ze_api.h | 7 -- 2 files changed, 4 insertions(+), 100 deletions(-) diff --git a/hwloc/topology-levelzero.c b/hwloc/topology-levelzero.c index 0252bae86..36af7c7bf 100644 --- a/hwloc/topology-levelzero.c +++ b/hwloc/topology-levelzero.c @@ -172,9 +172,9 @@ hwloc__levelzero_cqprops_get(ze_device_handle_t zeh, } static int -hwloc__levelzero_memory_get_from_sysman(zes_device_handle_t zesh, - hwloc_obj_t root_osdev, - unsigned nr_osdevs, hwloc_obj_t *sub_osdevs) +hwloc__levelzero_memory_get(zes_device_handle_t zesh, + hwloc_obj_t root_osdev, + unsigned nr_osdevs, hwloc_obj_t *sub_osdevs) { zes_mem_handle_t *mh; uint32_t nr_mems; @@ -279,95 +279,6 @@ hwloc__levelzero_memory_get_from_sysman(zes_device_handle_t zesh, return 0; } -static void -hwloc__levelzero_memory_get_from_coreapi(ze_device_handle_t zeh, - hwloc_obj_t osdev, - int ignore_ddr) -{ - ze_device_memory_properties_t *mh; - uint32_t nr_mems; - ze_result_t res; - - nr_mems = 0; - res = zeDeviceGetMemoryProperties(zeh, &nr_mems, NULL); - if (res != ZE_RESULT_SUCCESS || !nr_mems) - return; - hwloc_debug("L0/CoreAPI: found %u memories in osdev %s\n", - nr_mems, osdev->name); - - mh = malloc(nr_mems * sizeof(*mh)); - if (mh) { - res = zeDeviceGetMemoryProperties(zeh, &nr_mems, mh); - if (res == ZE_RESULT_SUCCESS) { - unsigned m; - for(m=0; mname); - if (!mh[m].totalSize) - continue; - if (ignore_ddr && !strcmp(_name, "DDR")) - continue; - if (!_name[0]) - _name = "Memory"; - snprintf(name, sizeof(name), "LevelZero%sSize", _name); /* HBM or DDR, or Memory if unknown */ - snprintf(value, sizeof(value), "%lluKiB", (unsigned long long) mh[m].totalSize >> 10); - hwloc_obj_add_info(osdev, name, value); - } - } - free(mh); - } -} - - -static void -hwloc__levelzero_memory_get(ze_device_handle_t zeh, zes_device_handle_t zesh, - hwloc_obj_t root_osdev, int is_integrated, - unsigned nr_subdevices, zes_device_handle_t *subzehs, hwloc_obj_t *sub_osdevs) -{ - static int memory_from_coreapi = -1; /* 1 means coreapi, 0 means sysman, -1 means sysman if available or coreapi otherwise */ - static int first = 1; - - if (first) { - char *env; - env = getenv("HWLOC_L0_COREAPI_MEMORY"); - if (env) - memory_from_coreapi = atoi(env); - - if (memory_from_coreapi == -1) { - int ret = hwloc__levelzero_memory_get_from_sysman(zesh, root_osdev, nr_subdevices, sub_osdevs); - if (!ret) { - /* sysman worked, we're done, disable coreapi for next time */ - hwloc_debug("levelzero: sysman/memory succeeded, disabling coreapi memory queries\n"); - memory_from_coreapi = 0; - return; - } - /* sysman failed, enable coreapi */ - hwloc_debug("levelzero: sysman/memory failed, enabling coreapi memory queries\n"); - memory_from_coreapi = 1; - } - - first = 0; - } - - if (memory_from_coreapi > 0) { - unsigned k; - int ignore_ddr = (memory_from_coreapi != 2) && is_integrated; /* DDR ignored in integrated GPUs, it's like the host DRAM */ - hwloc__levelzero_memory_get_from_coreapi(zeh, root_osdev, ignore_ddr); - for(k=0; k Date: Mon, 12 Jun 2023 11:12:08 +0200 Subject: [PATCH 2/4] levelzero: only get pci locality info from sysman Now that zesInit() is mandatory, don't bother trying the core API extension first in case sysman wouldn't be available. Hence remove the zeDevicePciGetPropertiesExt() optional detection Signed-off-by: Brice Goglin --- config/hwloc.m4 | 2 -- hwloc/topology-levelzero.c | 21 ------------------- tests/hwloc/ports/Makefile.am | 3 +-- .../include/levelzero/level_zero/ze_api.h | 21 ------------------- 4 files changed, 1 insertion(+), 46 deletions(-) diff --git a/config/hwloc.m4 b/config/hwloc.m4 index 4e9560f04..9df69892a 100644 --- a/config/hwloc.m4 +++ b/config/hwloc.m4 @@ -1413,7 +1413,6 @@ return clGetDeviceIDs(0, 0, 0, NULL, NULL); HWLOC_PKG_CHECK_MODULES([LEVELZERO], [libze_loader], [zesDriverGetDeviceByUuidExp], [level_zero/zes_api.h], [hwloc_levelzero_happy=yes HWLOC_LEVELZERO_REQUIRES=libze_loader - AC_CHECK_LIB([ze_loader], [zeDevicePciGetPropertiesExt], [AC_DEFINE(HWLOC_HAVE_ZEDEVICEPCIGETPROPERTIESEXT, 1, [Define to 1 if zeDevicePciGetPropertiesExt is available])]) ], [hwloc_levelzero_happy=no]) if test x$hwloc_levelzero_happy = xno; then hwloc_levelzero_happy=yes @@ -1423,7 +1422,6 @@ return clGetDeviceIDs(0, 0, 0, NULL, NULL); AC_CHECK_LIB([ze_loader], [zesDriverGetDeviceByUuidExp], [HWLOC_LEVELZERO_LIBS="-lze_loader" - AC_CHECK_LIB([ze_loader], [zeDevicePciGetPropertiesExt], [AC_DEFINE(HWLOC_HAVE_ZEDEVICEPCIGETPROPERTIESEXT, 1, [Define to 1 if zeDevicePciGetPropertiesExt is available])]) ], [hwloc_levelzero_happy=no]) ], [hwloc_levelzero_happy=no]) ], [hwloc_levelzero_happy=no]) diff --git a/hwloc/topology-levelzero.c b/hwloc/topology-levelzero.c index 36af7c7bf..b1426c9f5 100644 --- a/hwloc/topology-levelzero.c +++ b/hwloc/topology-levelzero.c @@ -559,26 +559,6 @@ hwloc__levelzero_devices_get(struct hwloc_topology *topology, hwloc__levelzero_ports_get(zesh, osdev, nr_subdevices, subosdevs, hports); parent = NULL; -#ifdef HWLOC_HAVE_ZEDEVICEPCIGETPROPERTIESEXT - { /* try getting PCI BDF+speed from core extension */ - ze_pci_ext_properties_t ext_pci; - ext_pci.stype = ZE_STRUCTURE_TYPE_PCI_EXT_PROPERTIES; - ext_pci.pNext = NULL; - res = zeDevicePciGetPropertiesExt(zeh, &ext_pci); - if (res == ZE_RESULT_SUCCESS) { - parent = hwloc_pci_find_parent_by_busid(topology, - ext_pci.address.domain, - ext_pci.address.bus, - ext_pci.address.device, - ext_pci.address.function); - if (parent && parent->type == HWLOC_OBJ_PCI_DEVICE) { - if (ext_pci.maxSpeed.maxBandwidth > 0) - parent->attr->pcidev.linkspeed = ((float)ext_pci.maxSpeed.maxBandwidth)/1000/1000/1000; - } - } - } -#endif /* HWLOC_HAVE_LEVELZERO_CORE_PCI_EXT */ - if (!parent) { /* try getting PCI BDF+speed from sysman */ zes_pci_properties_t pci; res = zesDevicePciGetProperties(zesh, &pci); @@ -593,7 +573,6 @@ hwloc__levelzero_devices_get(struct hwloc_topology *topology, parent->attr->pcidev.linkspeed = ((float)pci.maxSpeed.maxBandwidth)/1000/1000/1000; } } - } if (!parent) parent = hwloc_get_root_obj(topology); diff --git a/tests/hwloc/ports/Makefile.am b/tests/hwloc/ports/Makefile.am index 645cd1d4c..8addaf2af 100644 --- a/tests/hwloc/ports/Makefile.am +++ b/tests/hwloc/ports/Makefile.am @@ -177,8 +177,7 @@ libhwloc_port_levelzero_la_SOURCES = \ include/levelzero/level_zero/ze_api.h \ include/levelzero/level_zero/zes_api.h libhwloc_port_levelzero_la_CPPFLAGS = $(common_CPPFLAGS) \ - -I$(HWLOC_top_srcdir)/tests/hwloc/ports/include/levelzero \ - -DHWLOC_HAVE_ZEDEVICEPCIGETPROPERTIESEXT=1 + -I$(HWLOC_top_srcdir)/tests/hwloc/ports/include/levelzero nodist_libhwloc_port_gl_la_SOURCES = topology-gl.c libhwloc_port_gl_la_SOURCES = \ diff --git a/tests/hwloc/ports/include/levelzero/level_zero/ze_api.h b/tests/hwloc/ports/include/levelzero/level_zero/ze_api.h index adbf84336..9ca3bd0f5 100644 --- a/tests/hwloc/ports/include/levelzero/level_zero/ze_api.h +++ b/tests/hwloc/ports/include/levelzero/level_zero/ze_api.h @@ -58,25 +58,4 @@ extern ze_result_t zeDeviceGetCommandQueueGroupProperties(ze_driver_handle_t, ui extern ze_result_t zeDeviceGetSubDevices(ze_device_handle_t, uint32_t *, ze_device_handle_t*); -typedef struct ze_pci_address_ext { - uint32_t domain, bus, device, function; -} ze_pci_address_ext_t; - -typedef struct ze_pci_speed_ext { - int64_t maxBandwidth; -} ze_pci_speed_ext_t; - -typedef int ze_structure_type_t; - -#define ZE_STRUCTURE_TYPE_PCI_EXT_PROPERTIES 0x10008 - -typedef struct ze_pci_ext_properties { - ze_structure_type_t stype; - void* pNext; - ze_pci_address_ext_t address; - ze_pci_speed_ext_t maxSpeed; -} ze_pci_ext_properties_t; - -extern ze_result_t zeDevicePciGetPropertiesExt(ze_device_handle_t, ze_pci_ext_properties_t *); - #endif /* HWLOC_PORT_L0_ZE_API_H */ From 70c8f46bb53146427175a2d2e30998726b9c54b3 Mon Sep 17 00:00:00 2001 From: Brice Goglin Date: Wed, 13 Nov 2024 16:27:13 +0100 Subject: [PATCH 3/4] levelzero: remove the is_integrated variable We don't need it anymore. Signed-off-by: Brice Goglin --- hwloc/topology-levelzero.c | 15 +++------------ .../ports/include/levelzero/level_zero/ze_api.h | 1 - 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/hwloc/topology-levelzero.c b/hwloc/topology-levelzero.c index b1426c9f5..b5e3988a6 100644 --- a/hwloc/topology-levelzero.c +++ b/hwloc/topology-levelzero.c @@ -63,14 +63,12 @@ hwloc__levelzero_osdev_array_find(struct hwloc_osdev_array *array, static void hwloc__levelzero_properties_get(ze_device_handle_t zeh, zes_device_handle_t zesh, - hwloc_obj_t osdev, - int *is_integrated_p) + hwloc_obj_t osdev) { ze_result_t res; ze_device_properties_t prop; zes_device_properties_t prop2; int is_subdevice = 0; - int is_integrated = 0; memset(&prop, 0, sizeof(prop)); res = zeDeviceGetProperties(zeh, &prop); @@ -110,14 +108,8 @@ hwloc__levelzero_properties_get(ze_device_handle_t zeh, zes_device_handle_t zesh if (prop.flags & ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE) is_subdevice = 1; - - if (prop.flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED) - is_integrated = 1; } - if (is_integrated_p) - *is_integrated_p = is_integrated; - if (is_subdevice) /* sysman API on subdevice returns the same as root device, and we don't need those duplicate attributes */ return; @@ -462,7 +454,6 @@ hwloc__levelzero_devices_get(struct hwloc_topology *topology, hwloc_obj_t osdev, parent, *subosdevs = NULL; ze_device_properties_t props; zes_uuid_t uuid; - int is_integrated = 0; ze_bool_t onSubdevice = 0; uint32_t subdeviceId = 0; @@ -494,7 +485,7 @@ hwloc__levelzero_devices_get(struct hwloc_topology *topology, snprintf(buffer, sizeof(buffer), "%u", j); hwloc_obj_add_info(osdev, "LevelZeroDriverDeviceIndex", buffer); - hwloc__levelzero_properties_get(zeh, zesh, osdev, &is_integrated); + hwloc__levelzero_properties_get(zeh, zesh, osdev); hwloc__levelzero_cqprops_get(zeh, osdev); @@ -538,7 +529,7 @@ hwloc__levelzero_devices_get(struct hwloc_topology *topology, snprintf(tmp, sizeof(tmp), "%u", k); hwloc_obj_add_info(subosdevs[k], "LevelZeroSubdeviceID", tmp); - hwloc__levelzero_properties_get(subzeh, subzesh, subosdevs[k], NULL); + hwloc__levelzero_properties_get(subzeh, subzesh, subosdevs[k]); hwloc__levelzero_cqprops_get(subzeh, subosdevs[k]); } diff --git a/tests/hwloc/ports/include/levelzero/level_zero/ze_api.h b/tests/hwloc/ports/include/levelzero/level_zero/ze_api.h index 9ca3bd0f5..dffb4683b 100644 --- a/tests/hwloc/ports/include/levelzero/level_zero/ze_api.h +++ b/tests/hwloc/ports/include/levelzero/level_zero/ze_api.h @@ -28,7 +28,6 @@ typedef enum _ze_device_type { ZE_DEVICE_TYPE_VPU = 5 } ze_device_type_t; -#define ZE_DEVICE_PROPERTY_FLAG_INTEGRATED (1<<0) #define ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE (1<<1) #define ZE_MAX_DEVICE_UUID_SIZE 16 From c2b2cfd726244e7327f42b7063e30a083982e12a Mon Sep 17 00:00:00 2001 From: Brice Goglin Date: Wed, 13 Nov 2024 16:33:16 +0100 Subject: [PATCH 4/4] levelzero: don't get device properties twice Signed-off-by: Brice Goglin --- hwloc/topology-levelzero.c | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/hwloc/topology-levelzero.c b/hwloc/topology-levelzero.c index b5e3988a6..14207e7c6 100644 --- a/hwloc/topology-levelzero.c +++ b/hwloc/topology-levelzero.c @@ -63,16 +63,21 @@ hwloc__levelzero_osdev_array_find(struct hwloc_osdev_array *array, static void hwloc__levelzero_properties_get(ze_device_handle_t zeh, zes_device_handle_t zesh, - hwloc_obj_t osdev) + hwloc_obj_t osdev, ze_device_properties_t *prop) { ze_result_t res; - ze_device_properties_t prop; + ze_device_properties_t _prop; zes_device_properties_t prop2; int is_subdevice = 0; - memset(&prop, 0, sizeof(prop)); - res = zeDeviceGetProperties(zeh, &prop); - if (res == ZE_RESULT_SUCCESS) { + if (!prop) { + /* no properties were given, get ours */ + memset(&_prop, 0, sizeof(_prop)); + res = zeDeviceGetProperties(zeh, &_prop); + if (res == ZE_RESULT_SUCCESS) + prop = &_prop; + } + if (prop) { /* name is the model name followed by the deviceID * flags 1<<0 means integrated (vs discrete). */ @@ -81,7 +86,7 @@ hwloc__levelzero_properties_get(ze_device_handle_t zeh, zes_device_handle_t zesh unsigned i; const char *type; - switch (prop.type) { + switch (prop->type) { case ZE_DEVICE_TYPE_GPU: type = "GPU"; break; case ZE_DEVICE_TYPE_CPU: type = "CPU"; break; case ZE_DEVICE_TYPE_FPGA: type = "FPGA"; break; @@ -89,24 +94,24 @@ hwloc__levelzero_properties_get(ze_device_handle_t zeh, zes_device_handle_t zesh case ZE_DEVICE_TYPE_VPU: type = "VPU"; break; default: if (HWLOC_SHOW_ALL_ERRORS()) - fprintf(stderr, "hwloc/levelzero: unexpected device type %u\n", (unsigned) prop.type); + fprintf(stderr, "hwloc/levelzero: unexpected device type %u\n", (unsigned) prop->type); type = "Unknown"; } hwloc_obj_add_info(osdev, "LevelZeroDeviceType", type); - snprintf(tmp, sizeof(tmp), "%u", prop.numSlices); + snprintf(tmp, sizeof(tmp), "%u", prop->numSlices); hwloc_obj_add_info(osdev, "LevelZeroNumSlices", tmp); - snprintf(tmp, sizeof(tmp), "%u", prop.numSubslicesPerSlice); + snprintf(tmp, sizeof(tmp), "%u", prop->numSubslicesPerSlice); hwloc_obj_add_info(osdev, "LevelZeroNumSubslicesPerSlice", tmp); - snprintf(tmp, sizeof(tmp), "%u", prop.numEUsPerSubslice); + snprintf(tmp, sizeof(tmp), "%u", prop->numEUsPerSubslice); hwloc_obj_add_info(osdev, "LevelZeroNumEUsPerSubslice", tmp); - snprintf(tmp, sizeof(tmp), "%u", prop.numThreadsPerEU); + snprintf(tmp, sizeof(tmp), "%u", prop->numThreadsPerEU); hwloc_obj_add_info(osdev, "LevelZeroNumThreadsPerEU", tmp); for(i=0; iuuid.id[i]); hwloc_obj_add_info(osdev, "LevelZeroUUID", uuid); - if (prop.flags & ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE) + if (prop->flags & ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE) is_subdevice = 1; } @@ -485,7 +490,7 @@ hwloc__levelzero_devices_get(struct hwloc_topology *topology, snprintf(buffer, sizeof(buffer), "%u", j); hwloc_obj_add_info(osdev, "LevelZeroDriverDeviceIndex", buffer); - hwloc__levelzero_properties_get(zeh, zesh, osdev); + hwloc__levelzero_properties_get(zeh, zesh, osdev, &props); hwloc__levelzero_cqprops_get(zeh, osdev); @@ -529,7 +534,7 @@ hwloc__levelzero_devices_get(struct hwloc_topology *topology, snprintf(tmp, sizeof(tmp), "%u", k); hwloc_obj_add_info(subosdevs[k], "LevelZeroSubdeviceID", tmp); - hwloc__levelzero_properties_get(subzeh, subzesh, subosdevs[k]); + hwloc__levelzero_properties_get(subzeh, subzesh, subosdevs[k], NULL); hwloc__levelzero_cqprops_get(subzeh, subosdevs[k]); }