Update hpctests to obey UCX_NET_DEVICES when RoCE devices present (#421)

* Turn off higher priority MPI net devices * Update pingmatrix.sh.j2 * Update pingmatrix.sh.j2 * Update pingpong.sh.j2 * Replace j2 comments with bash * Update pingpong.sh.j2 --------- Co-authored-by: Steve Brasier <[email protected]>
stackhpc · Aug 7, 2024 · bd2de50 · bd2de50
1 parent 81f7de6
commit bd2de50
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 2 deletions.
diff --git a/ansible/roles/hpctests/templates/pingmatrix.sh.j2 b/ansible/roles/hpctests/templates/pingmatrix.sh.j2
@@ -16,4 +16,8 @@ echo UCX_NET_DEVICES: $UCX_NET_DEVICES
 module load {{ hpctests_pingmatrix_modules | join(' ' ) }}
 
 mpicc -o nxnlatbw mpi_nxnlatbw.c
-mpirun nxnlatbw
+
+# mpirun flags force using UCX TCP transports, overriding higher
+# priority of OpenMPI btl/openib component, which is also using RDMA
+# https://wiki.stackhpc.com/s/985dae84-7bd8-4924-94b7-9629a7827100 
+mpirun -mca pml_ucx_tls any -mca pml_ucx_devices any nxnlatbw
diff --git a/ansible/roles/hpctests/templates/pingpong.sh.j2 b/ansible/roles/hpctests/templates/pingpong.sh.j2
@@ -16,4 +16,8 @@ echo UCX_NET_DEVICES: $UCX_NET_DEVICES
 module load {{ hpctests_pingpong_modules | join(' ' ) }}
 
 #srun --mpi=pmi2 IMB-MPI1 pingpong # doesn't work in ohpc v2.1
-mpirun IMB-MPI1 pingpong
+
+# mpirun flags force using UCX TCP transports, overriding higher 
+# priority of OpenMPI btl/openib component, which is also using RDMA 
+# https://wiki.stackhpc.com/s/985dae84-7bd8-4924-94b7-9629a7827100
+mpirun -mca pml_ucx_tls any -mca pml_ucx_devices any IMB-MPI1 pingpong