TL/MLX5: Add configuration to set IB QP SL

openucx · Dec 17, 2024 · d6673c8 · d6673c8
1 parent 73651ea
commit d6673c8
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 1 deletion.
diff --git a/src/components/tl/mlx5/tl_mlx5.c b/src/components/tl/mlx5/tl_mlx5.c
@@ -67,6 +67,10 @@ static ucc_config_field_t ucc_tl_mlx5_lib_config_table[] = {
      ucc_offsetof(ucc_tl_mlx5_lib_config_t, qp_conf.qp_max_atomic),
      UCC_CONFIG_TYPE_UINT},
 
+     {"QP_SL", "0", "IB QP Service Level",
+     ucc_offsetof(ucc_tl_mlx5_lib_config_t, qp_conf.qp_sl),
+     UCC_CONFIG_TYPE_UINT},
+
     {"MCAST_SX_DEPTH", "512", "Send context depth of the Mcast comm",
      ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.sx_depth),
      UCC_CONFIG_TYPE_INT},

diff --git a/src/components/tl/mlx5/tl_mlx5.h b/src/components/tl/mlx5/tl_mlx5.h
@@ -42,6 +42,7 @@ typedef struct ucc_tl_mlx5_iface {
 extern ucc_tl_mlx5_iface_t ucc_tl_mlx5;
 
 typedef struct ucc_tl_mlx5_ib_qp_conf {
+    uint8_t             qp_sl;
     uint32_t            qp_rnr_retry;
     uint32_t            qp_rnr_timer;
     uint32_t            qp_retry_cnt;

diff --git a/src/components/tl/mlx5/tl_mlx5_ib.c b/src/components/tl/mlx5/tl_mlx5_ib.c
@@ -141,7 +141,7 @@ ucc_status_t ucc_tl_mlx5_qp_connect(struct ibv_qp *qp, uint32_t qp_num,
     qp_attr.min_rnr_timer         = qp_conf->qp_rnr_timer;
     qp_attr.max_dest_rd_atomic    = qp_conf->qp_max_atomic;
     qp_attr.ah_attr.dlid          = lid;
-    qp_attr.ah_attr.sl            = 0;
+    qp_attr.ah_attr.sl            = qp_conf->qp_sl;
     qp_attr.ah_attr.src_path_bits = 0;
     qp_attr.ah_attr.port_num      = port;
 
@@ -199,6 +199,7 @@ ucc_status_t ucc_tl_mlx5_init_dct(struct ibv_pd *pd, struct ibv_context *ctx,
     qp_attr_to_rtr.min_rnr_timer     = qp_conf->qp_rnr_timer;
     qp_attr_to_rtr.ah_attr.port_num  = port_num;
     qp_attr_to_rtr.ah_attr.is_global = 0;
+    qp_attr_to_rtr.ah_attr.sl        = qp_conf->qp_sl;
 
     attr_ex.qp_type = IBV_QPT_DRIVER;
     attr_ex.send_cq = cq;
@@ -289,6 +290,7 @@ ucc_status_t ucc_tl_mlx5_init_dci(ucc_tl_mlx5_dci_t *dci, struct ibv_pd *pd,
     qp_attr_to_rtr.min_rnr_timer     = qp_conf->qp_rnr_timer;
     qp_attr_to_rtr.ah_attr.port_num  = port_num;
     qp_attr_to_rtr.ah_attr.is_global = 0;
+    qp_attr_to_rtr.ah_attr.sl        = qp_conf->qp_sl;
 
     qp_attr_to_rts.qp_state      = IBV_QPS_RTS;
     qp_attr_to_rts.timeout       = qp_conf->qp_timeout;

diff --git a/test/gtest/tl/mlx5/test_tl_mlx5_qps.h b/test/gtest/tl/mlx5/test_tl_mlx5_qps.h
@@ -18,6 +18,7 @@ class test_tl_mlx5_qp : public test_tl_mlx5 {
         qp_conf.qp_retry_cnt  = 7;
         qp_conf.qp_timeout    = 18;
         qp_conf.qp_max_atomic = 1;
+        qp_conf.qp_sl         = 1;
     }
 };