Merge pull request #6910 from yosefe/topic/uct-dc-fix-setting-dscp-va…

…lue-for UCT/DC: Fix setting DSCP value for RoCE DCT
openucx · Jun 6, 2021 · d6d412e · d6d412e
2 parents e9d9b05 + 0a1936b
commit d6d412e
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 2 deletions.
diff --git a/src/uct/ib/base/ib_iface.h b/src/uct/ib/base/ib_iface.h
@@ -629,4 +629,11 @@ uct_ib_cq_size(uct_ib_iface_t *iface, const uct_ib_iface_init_attr_t *init_attr,
     }
 }
 
+static UCS_F_ALWAYS_INLINE unsigned
+uct_ib_iface_roce_dscp(uct_ib_iface_t *iface)
+{
+    ucs_assert(uct_ib_iface_is_roce(iface));
+    return iface->config.traffic_class >> 2;
+}
+
 #endif
diff --git a/src/uct/ib/dc/dc_mlx5_devx.c b/src/uct/ib/dc/dc_mlx5_devx.c
@@ -56,7 +56,17 @@ ucs_status_t uct_dc_mlx5_iface_devx_create_dct(uct_dc_mlx5_iface_t *iface,
     UCT_IB_MLX5DV_SET(dctc, dctc, port, iface->super.super.super.config.port_num);
 
     UCT_IB_MLX5DV_SET(dctc, dctc, min_rnr_nak, iface->super.super.config.min_rnr_timer);
-    UCT_IB_MLX5DV_SET(dctc, dctc, tclass, iface->super.super.super.config.traffic_class);
+
+    if (uct_ib_iface_is_roce_v2(&iface->super.super.super, dev)) {
+        /* RoCE V2 sets DSCP */
+        UCT_IB_MLX5DV_SET(dctc, dctc, dscp,
+                          uct_ib_iface_roce_dscp(&iface->super.super.super));
+    } else if (!uct_ib_iface_is_roce(&iface->super.super.super)) {
+        /* Infiniband sets traffic class */
+        UCT_IB_MLX5DV_SET(dctc, dctc, tclass,
+                          iface->super.super.super.config.traffic_class);
+    }
+
     UCT_IB_MLX5DV_SET(dctc, dctc, mtu, iface->super.super.super.config.path_mtu);
     UCT_IB_MLX5DV_SET(dctc, dctc, my_addr_index, iface->super.super.super.gid_info.gid_index);
     UCT_IB_MLX5DV_SET(dctc, dctc, hop_limit, iface->super.super.super.config.hop_limit);

diff --git a/src/uct/ib/rc/accel/rc_mlx5_devx.c b/src/uct/ib/rc/accel/rc_mlx5_devx.c
@@ -377,7 +377,7 @@ uct_rc_mlx5_iface_common_devx_connect_qp(uct_rc_mlx5_iface_common_t *iface,
             UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.udp_sport,
                               ah_attr->dlid);
             UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.dscp,
-                              iface->super.super.config.traffic_class >> 2);
+                              uct_ib_iface_roce_dscp(&iface->super.super));
         }
 
         uct_ib_mlx5_devx_set_qpc_port_affinity(md, path_index, qpc,