Merge pull request #10364 from roiedanino/v1.18.x/ddp-devx

v1.18.x - UCT/IB/MLX5: Add DDP support for DevX

Author: yosefe

src/ucs/sys/compiler_def.h

@@ -80,6 +80,9 @@
 /* Mask of bits 0..i-1 */
 #define UCS_MASK(_i)             (((_i) >= 64) ? ~0 : (UCS_BIT(_i) - 1))
 
+/* The i-th bit */
+#define UCS_BIT_GET(_value, _i)  (!!((_value) & UCS_BIT(_i)))
+
 /*
  * Enable compiler checks for printf-like formatting.
  *

src/uct/ib/base/ib_iface.h

@@ -239,7 +239,9 @@ enum {
     /* Indicates that TX cq len in uct_ib_iface_init_attr_t is specified per
      * each IB path. Therefore IB interface constructor would need to multiply
      * TX CQ len by the number of IB paths (when it is properly initialized). */
-    UCT_IB_TX_OPS_PER_PATH           = UCS_BIT(2)
+    UCT_IB_TX_OPS_PER_PATH           = UCS_BIT(2),
+    /* Whether device and transport supports DDP */
+    UCT_IB_DDP_SUPPORTED             = UCS_BIT(3)
 };
 
 
@@ -345,7 +347,6 @@ struct uct_ib_iface {
         enum ibv_mtu                 path_mtu;
         uint8_t                      counter_set_id;
         uct_ib_iface_send_overhead_t send_overhead;
-        ucs_ternary_auto_value_t     dp_ordering_ooo; /* Activate RW OOO */
     } config;
 
     uct_ib_iface_ops_t        *ops;

src/uct/ib/mlx5/dc/dc_mlx5.c

@@ -1610,6 +1610,10 @@ static UCS_CLASS_INIT_FUNC(uct_dc_mlx5_iface_t, uct_md_h tl_md, uct_worker_h wor
         init_attr.flags  |= UCT_IB_TM_SUPPORTED;
     }
 
+    if (md->dp_ordering_cap.dc == UCT_IB_MLX5_DP_ORDERING_OOO_ALL) {
+        init_attr.flags |= UCT_IB_DDP_SUPPORTED;
+    }
+
     status = uct_dc_mlx5_calc_sq_length(md, tx_queue_len, &sq_length);
     if (status != UCS_OK) {
         return status;
@@ -1623,9 +1627,9 @@ static UCS_CLASS_INIT_FUNC(uct_dc_mlx5_iface_t, uct_md_h tl_md, uct_worker_h wor
                               tl_md, worker, params, &config->super,
                               &config->rc_mlx5_common, &init_attr);
 
-    status = uct_rc_mlx5_dp_ordering_ooo_init(
-            &self->super, UCT_IB_MLX5_MD_FLAG_DP_ORDERING_OOO_RW_DC,
-            &config->rc_mlx5_common, "dc");
+    status = uct_rc_mlx5_dp_ordering_ooo_init(&self->super,
+                                              md->dp_ordering_cap.dc,
+                                              &config->rc_mlx5_common, "dc");
     if (status != UCS_OK) {
         return status;
     }

src/uct/ib/mlx5/dc/dc_mlx5_devx.c

@@ -49,15 +49,14 @@ ucs_status_t uct_dc_mlx5_iface_devx_create_dct(uct_dc_mlx5_iface_t *iface)
                               ib_iface->config.max_inl_cqe[UCT_IB_DIR_RX], 1));
     UCT_IB_MLX5DV_SET(dctc, dctc, atomic_mode,
                       uct_ib_mlx5_get_atomic_mode(ib_iface));
-    if (uct_ib_iface_is_roce(&iface->super.super.super)) {
-        UCT_IB_MLX5DV_SET(dctc, dctc, dp_ordering_0,
-                          ucs_ternary_auto_value_is_yes_or_try(
-                                  ib_iface->config.dp_ordering_ooo));
-        UCT_IB_MLX5DV_SET(dctc, dctc, dp_ordering_1, 0);
-        UCT_IB_MLX5DV_SET(dctc, dctc, dp_ordering_force,
-                          ucs_ternary_auto_value_is_yes_or_no(
-                                  ib_iface->config.dp_ordering_ooo));
-    } else {
+    UCT_IB_MLX5DV_SET(dctc, dctc, dp_ordering_0,
+                      UCS_BIT_GET(iface->super.config.dp_ordering, 0));
+    UCT_IB_MLX5DV_SET(dctc, dctc, dp_ordering_1,
+                      UCS_BIT_GET(iface->super.config.dp_ordering, 1));
+    UCT_IB_MLX5DV_SET(dctc, dctc, dp_ordering_force,
+                      iface->super.config.dp_ordering_force);
+
+    if (!uct_ib_iface_is_roce(&iface->super.super.super)) {
         UCT_IB_MLX5DV_SET(dctc, dctc, pkey_index, ib_iface->pkey_index);
     }
 
@@ -145,16 +144,16 @@ ucs_status_t uct_dc_mlx5_iface_devx_dci_connect(uct_dc_mlx5_iface_t *iface,
     UCT_IB_MLX5DV_SET(qpc, qpc, atomic_mode,
                       uct_ib_mlx5_get_atomic_mode(&rc_iface->super));
     UCT_IB_MLX5DV_SET(qpc, qpc, rae, true);
+
+    uct_ib_mlx5_devx_set_qpc_dp_ordering(md, qpc, &iface->super);
+
     if (uct_ib_iface_is_roce(&rc_iface->super)) {
         UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.eth_prio,
                           rc_iface->super.config.sl);
         if (iface->tx.port_affinity) {
             uct_ib_mlx5_devx_set_qpc_port_affinity(md, dci_config->path_index,
                                                    qpc, &opt_param_mask);
         }
-
-        uct_ib_mlx5_devx_set_qpc_dp_ordering(
-                qpc, rc_iface->super.config.dp_ordering_ooo);
     } else {
         UCT_IB_MLX5DV_SET(qpc, qpc, primary_address_path.sl,
                           rc_iface->super.config.sl);

src/uct/ib/mlx5/dv/ib_mlx5_dv.c

@@ -462,16 +462,6 @@ ucs_status_t uct_ib_mlx5_devx_query_ooo_sl_mask(uct_ib_mlx5_md_t *md,
     return UCS_OK;
 }
 
-void uct_ib_mlx5_devx_set_qpc_dp_ordering(
-        void *qpc, ucs_ternary_auto_value_t dp_ordering_ooo)
-{
-    UCT_IB_MLX5DV_SET(qpc, qpc, dp_ordering_0,
-                      ucs_ternary_auto_value_is_yes_or_try(dp_ordering_ooo));
-    UCT_IB_MLX5DV_SET(qpc, qpc, dp_ordering_1, 0);
-    UCT_IB_MLX5DV_SET(qpc, qpc, dp_ordering_force,
-                      ucs_ternary_auto_value_is_yes_or_no(dp_ordering_ooo));
-}
-
 void uct_ib_mlx5_devx_set_qpc_port_affinity(uct_ib_mlx5_md_t *md,
                                             uint8_t path_index, void *qpc,
                                             uint32_t *opt_param_mask)

src/uct/ib/mlx5/dv/ib_mlx5_ifc.h

@@ -323,7 +323,12 @@ struct uct_ib_mlx5_cmd_hca_cap_bits {
     uint8_t    log_max_transport_domain[0x5];
     uint8_t    reserved_at_328[0x3];
     uint8_t    log_max_pd[0x5];
-    uint8_t    reserved_at_330[0xb];
+    uint8_t    dp_ordering_ooo_all_ud[0x1];
+    uint8_t    dp_ordering_ooo_all_uc[0x1];
+    uint8_t    dp_ordering_ooo_all_xrc[0x1];
+    uint8_t    dp_ordering_ooo_all_dc[0x1];
+    uint8_t    dp_ordering_ooo_all_rc[0x1];
+    uint8_t    reserved_at_335[0x6];
     uint8_t    log_max_xrcd[0x5];
 
     uint8_t    nic_receive_steering_discard[0x1];

src/uct/ib/mlx5/dv/ib_mlx5dv_md.c

@@ -1883,12 +1883,20 @@ static void uct_ib_mlx5_devx_check_dp_ordering(uct_ib_mlx5_md_t *md, void *cap,
                                                void *cap_2,
                                                uct_ib_device_t *dev)
 {
-    if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, dp_ordering_ooo_rw_rc)) {
-        md->flags |= UCT_IB_MLX5_MD_FLAG_DP_ORDERING_OOO_RW_RC;
+    if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, dp_ordering_ooo_all_rc)) {
+        md->dp_ordering_cap.rc = UCT_IB_MLX5_DP_ORDERING_OOO_ALL;
+    } else if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, dp_ordering_ooo_rw_rc)) {
+        md->dp_ordering_cap.rc = UCT_IB_MLX5_DP_ORDERING_OOO_RW;
+    } else {
+        md->dp_ordering_cap.rc = UCT_IB_MLX5_DP_ORDERING_IBTA;
     }
 
-    if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, dp_ordering_ooo_rw_dc)) {
-        md->flags |= UCT_IB_MLX5_MD_FLAG_DP_ORDERING_OOO_RW_DC;
+    if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, dp_ordering_ooo_all_dc)) {
+        md->dp_ordering_cap.dc = UCT_IB_MLX5_DP_ORDERING_OOO_ALL;
+    } else if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, dp_ordering_ooo_rw_dc)) {
+        md->dp_ordering_cap.dc = UCT_IB_MLX5_DP_ORDERING_OOO_RW;
+    } else {
+        md->dp_ordering_cap.dc = UCT_IB_MLX5_DP_ORDERING_IBTA;
     }
 
     if ((cap_2 != NULL) &&
@@ -1899,8 +1907,7 @@ static void uct_ib_mlx5_devx_check_dp_ordering(uct_ib_mlx5_md_t *md, void *cap,
     ucs_debug("%s: dp_ordering support: force=%d ooo_rw_rc=%d ooo_rw_dc=%d",
               uct_ib_device_name(dev),
               !!(md->flags & UCT_IB_MLX5_MD_FLAG_DP_ORDERING_FORCE),
-              !!(md->flags & UCT_IB_MLX5_MD_FLAG_DP_ORDERING_OOO_RW_RC),
-              !!(md->flags & UCT_IB_MLX5_MD_FLAG_DP_ORDERING_OOO_RW_DC));
+              md->dp_ordering_cap.rc, md->dp_ordering_cap.dc);
 }
 
 static void uct_ib_mlx5_devx_check_mkey_by_name(uct_ib_mlx5_md_t *md,
@@ -3153,7 +3160,8 @@ static void uct_ib_mlx5dv_check_dc(uct_ib_device_t *dev)
 
 static uct_ib_md_ops_t uct_ib_mlx5_md_ops;
 
-static void uct_ib_mlx5dv_query_ddp(struct ibv_context *ctx, uct_ib_mlx5_md_t *md)
+static ucs_status_t 
+uct_ib_mlx5dv_check_ddp(struct ibv_context *ctx, uct_ib_mlx5_md_t *md)
 {
 #ifdef HAVE_OOO_RECV_WRS
     struct mlx5dv_context ctx_dv = {
@@ -3164,13 +3172,21 @@ static void uct_ib_mlx5dv_query_ddp(struct ibv_context *ctx, uct_ib_mlx5_md_t *m
     ret = mlx5dv_query_device(ctx, &ctx_dv);
     if (ret != 0) {
         ucs_error("mlx5dv_query_device: Failed to query device capabilities, ret=%d\n", ret);
-        return;
+        return UCS_ERR_NO_RESOURCE;
     }
 
     if (ctx_dv.ooo_recv_wrs_caps.max_rc > 0) {
-        md->flags |= UCT_IB_MLX5_MD_FLAG_DDP;
+        md->dp_ordering_cap.rc = UCT_IB_MLX5_DP_ORDERING_OOO_ALL;
+    }
+
+    if (ctx_dv.ooo_recv_wrs_caps.max_dct > 0) {
+        md->dp_ordering_cap.dc = UCT_IB_MLX5_DP_ORDERING_OOO_ALL;
     }
+#else
+    md->dp_ordering_cap.rc = UCT_IB_MLX5_DP_ORDERING_IBTA;
+    md->dp_ordering_cap.dc = UCT_IB_MLX5_DP_ORDERING_IBTA;
 #endif
+    return UCS_OK;
 }
 
 static ucs_status_t uct_ib_mlx5dv_md_open(struct ibv_device *ibv_device,
@@ -3208,7 +3224,10 @@ static ucs_status_t uct_ib_mlx5dv_md_open(struct ibv_device *ibv_device,
         goto err_md_free;
     }
 
-    uct_ib_mlx5dv_query_ddp(ctx, md);
+    status = uct_ib_mlx5dv_check_ddp(ctx, md);
+    if (status != UCS_OK) {
+        goto err_md_free;
+    }
 
     if (IBV_DEVICE_ATOMIC_HCA(dev)) {
         dev->atomic_arg_sizes = sizeof(uint64_t);

src/uct/ib/mlx5/gga/gga_mlx5.c

@@ -663,7 +663,8 @@ static UCS_CLASS_INIT_FUNC(uct_gga_mlx5_iface_t,
                               &init_attr);
 
     status = uct_rc_mlx5_dp_ordering_ooo_init(
-            &self->super, UCT_IB_MLX5_MD_FLAG_DP_ORDERING_OOO_RW_RC,
+            &self->super,
+            ucs_min(md->dp_ordering_cap.rc, UCT_IB_MLX5_DP_ORDERING_OOO_RW),
             &config->rc_mlx5_common, "gga");
     if (status != UCS_OK) {
         return status;

src/uct/ib/mlx5/ib_mlx5.h

@@ -204,17 +204,11 @@ enum {
     UCT_IB_MLX5_MD_FLAG_UAR_USE_WC           = UCS_BIT(17),
     /* Device supports implicit ODP with PCI relaxed order */
     UCT_IB_MLX5_MD_FLAG_GVA_RO               = UCS_BIT(18),
-    /* RoCE supports out-of-order RDMA for RC */
-    UCT_IB_MLX5_MD_FLAG_DP_ORDERING_OOO_RW_RC = UCS_BIT(19),
-    /* RoCE supports out-of-order RDMA for DC */
-    UCT_IB_MLX5_MD_FLAG_DP_ORDERING_OOO_RW_DC = UCS_BIT(20),
-    /* RoCE supports forcing ordering configuration */
-    UCT_IB_MLX5_MD_FLAG_DP_ORDERING_FORCE     = UCS_BIT(21),
-    /* Device supports DDP (OOO data placement)*/
-    UCT_IB_MLX5_MD_FLAG_DDP                   = UCS_BIT(22),
+    /* Device supports forcing ordering configuration */
+    UCT_IB_MLX5_MD_FLAG_DP_ORDERING_FORCE     = UCS_BIT(19),
 
     /* Object to be created by DevX */
-    UCT_IB_MLX5_MD_FLAG_DEVX_OBJS_SHIFT  = 23,
+    UCT_IB_MLX5_MD_FLAG_DEVX_OBJS_SHIFT  = 20,
     UCT_IB_MLX5_MD_FLAG_DEVX_RC_QP       = UCT_IB_MLX5_MD_FLAG_DEVX_OBJS(RCQP),
     UCT_IB_MLX5_MD_FLAG_DEVX_RC_SRQ      = UCT_IB_MLX5_MD_FLAG_DEVX_OBJS(RCSRQ),
     UCT_IB_MLX5_MD_FLAG_DEVX_DCT         = UCT_IB_MLX5_MD_FLAG_DEVX_OBJS(DCT),
@@ -386,6 +380,16 @@ KHASH_MAP_INIT_INT(rkeys, uct_ib_mlx5_mem_lru_entry_t*);
 #endif
 
 
+typedef enum {
+    /* IBTA-compliant ordering semantics */
+    UCT_IB_MLX5_DP_ORDERING_IBTA    = 0x0,
+    /* Out-of-order RDMA reads and writes */
+    UCT_IB_MLX5_DP_ORDERING_OOO_RW  = 0x1,
+    /* Out-of-order RDMA read/write/send/recv (DDP) */
+    UCT_IB_MLX5_DP_ORDERING_OOO_ALL = 0x2,
+} uct_ib_mlx5_dp_ordering_t;
+
+
 /**
  * MLX5 IB memory domain.
  */
@@ -431,6 +435,12 @@ typedef struct uct_ib_mlx5_md {
     uint8_t                  max_rd_atomic_dc;
     uint8_t                  log_max_dci_stream_channels;
     uint32_t                 smkey_index;
+    struct {
+        /* Max dp ordering level per transport, 
+           as listed in uct_ib_mlx5_dp_ordering_t */
+        uint8_t              rc;
+        uint8_t              dc;
+    } dp_ordering_cap;
 } uct_ib_mlx5_md_t;
 
 
@@ -986,9 +996,6 @@ ucs_status_t uct_ib_mlx5_devx_query_ooo_sl_mask(uct_ib_mlx5_md_t *md,
                                                 uint8_t port_num,
                                                 uint16_t *ooo_sl_mask_p);
 
-void uct_ib_mlx5_devx_set_qpc_dp_ordering(
-        void *qpc, ucs_ternary_auto_value_t dp_ordering_ooo);
-
 void uct_ib_mlx5_devx_set_qpc_port_affinity(uct_ib_mlx5_md_t *md,
                                             uint8_t path_index, void *qpc,
                                             uint32_t *opt_param_mask);