From d6673c84b9309bbaa208290a95826296e166352a Mon Sep 17 00:00:00 2001 From: dmitrygx Date: Wed, 11 Dec 2024 12:52:40 +0300 Subject: [PATCH] TL/MLX5: Add configuration to set IB QP SL --- src/components/tl/mlx5/tl_mlx5.c | 4 ++++ src/components/tl/mlx5/tl_mlx5.h | 1 + src/components/tl/mlx5/tl_mlx5_ib.c | 4 +++- test/gtest/tl/mlx5/test_tl_mlx5_qps.h | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/components/tl/mlx5/tl_mlx5.c b/src/components/tl/mlx5/tl_mlx5.c index 5cdd6c51a1..6b481c0392 100644 --- a/src/components/tl/mlx5/tl_mlx5.c +++ b/src/components/tl/mlx5/tl_mlx5.c @@ -67,6 +67,10 @@ static ucc_config_field_t ucc_tl_mlx5_lib_config_table[] = { ucc_offsetof(ucc_tl_mlx5_lib_config_t, qp_conf.qp_max_atomic), UCC_CONFIG_TYPE_UINT}, + {"QP_SL", "0", "IB QP Service Level", + ucc_offsetof(ucc_tl_mlx5_lib_config_t, qp_conf.qp_sl), + UCC_CONFIG_TYPE_UINT}, + {"MCAST_SX_DEPTH", "512", "Send context depth of the Mcast comm", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.sx_depth), UCC_CONFIG_TYPE_INT}, diff --git a/src/components/tl/mlx5/tl_mlx5.h b/src/components/tl/mlx5/tl_mlx5.h index 159ecda8ed..2ca6acd785 100644 --- a/src/components/tl/mlx5/tl_mlx5.h +++ b/src/components/tl/mlx5/tl_mlx5.h @@ -42,6 +42,7 @@ typedef struct ucc_tl_mlx5_iface { extern ucc_tl_mlx5_iface_t ucc_tl_mlx5; typedef struct ucc_tl_mlx5_ib_qp_conf { + uint8_t qp_sl; uint32_t qp_rnr_retry; uint32_t qp_rnr_timer; uint32_t qp_retry_cnt; diff --git a/src/components/tl/mlx5/tl_mlx5_ib.c b/src/components/tl/mlx5/tl_mlx5_ib.c index a34df036f3..9484b0eac9 100644 --- a/src/components/tl/mlx5/tl_mlx5_ib.c +++ b/src/components/tl/mlx5/tl_mlx5_ib.c @@ -141,7 +141,7 @@ ucc_status_t ucc_tl_mlx5_qp_connect(struct ibv_qp *qp, uint32_t qp_num, qp_attr.min_rnr_timer = qp_conf->qp_rnr_timer; qp_attr.max_dest_rd_atomic = qp_conf->qp_max_atomic; qp_attr.ah_attr.dlid = lid; - qp_attr.ah_attr.sl = 0; + qp_attr.ah_attr.sl = qp_conf->qp_sl; qp_attr.ah_attr.src_path_bits = 0; qp_attr.ah_attr.port_num = port; @@ -199,6 +199,7 @@ ucc_status_t ucc_tl_mlx5_init_dct(struct ibv_pd *pd, struct ibv_context *ctx, qp_attr_to_rtr.min_rnr_timer = qp_conf->qp_rnr_timer; qp_attr_to_rtr.ah_attr.port_num = port_num; qp_attr_to_rtr.ah_attr.is_global = 0; + qp_attr_to_rtr.ah_attr.sl = qp_conf->qp_sl; attr_ex.qp_type = IBV_QPT_DRIVER; attr_ex.send_cq = cq; @@ -289,6 +290,7 @@ ucc_status_t ucc_tl_mlx5_init_dci(ucc_tl_mlx5_dci_t *dci, struct ibv_pd *pd, qp_attr_to_rtr.min_rnr_timer = qp_conf->qp_rnr_timer; qp_attr_to_rtr.ah_attr.port_num = port_num; qp_attr_to_rtr.ah_attr.is_global = 0; + qp_attr_to_rtr.ah_attr.sl = qp_conf->qp_sl; qp_attr_to_rts.qp_state = IBV_QPS_RTS; qp_attr_to_rts.timeout = qp_conf->qp_timeout; diff --git a/test/gtest/tl/mlx5/test_tl_mlx5_qps.h b/test/gtest/tl/mlx5/test_tl_mlx5_qps.h index d80776e370..f4b87df5c7 100644 --- a/test/gtest/tl/mlx5/test_tl_mlx5_qps.h +++ b/test/gtest/tl/mlx5/test_tl_mlx5_qps.h @@ -18,6 +18,7 @@ class test_tl_mlx5_qp : public test_tl_mlx5 { qp_conf.qp_retry_cnt = 7; qp_conf.qp_timeout = 18; qp_conf.qp_max_atomic = 1; + qp_conf.qp_sl = 1; } };