[v1.14.x] prov/rxm: Always use rendezvous protocol for ZE device memo…

…ry send Bounce buffer copy overhead is high for ZE device memory. The rendezvous protocol takes advantage of GPU RDMA and performs better even for small messages. Signed-off-by: Jianxin Xiong <[email protected]> (cherry-picked an adapted from commit 7efb4f8)
HewlettPackard · Dec 7, 2022 · 6fd2a41 · 6fd2a41
1 parent d93b078
commit 6fd2a41
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/prov/rxm/src/rxm_ep.c b/prov/rxm/src/rxm_ep.c
@@ -1525,6 +1525,10 @@ rxm_send_common(struct rxm_ep *rxm_ep, struct rxm_conn *rxm_conn,
 		(data_len > rxm_ep->rxm_info->tx_attr->inject_size)) ||
 	       (data_len <= rxm_ep->rxm_info->tx_attr->inject_size));
 
+	iface = rxm_mr_desc_to_hmem_iface_dev(desc, count, &device);
+	if (iface == FI_HMEM_ZE)
+		goto rndv_send;
+
 	if (data_len <= rxm_ep->eager_limit) {
 		ret = rxm_send_eager(rxm_ep, rxm_conn, iov, desc, count,
 				     context, data, flags, tag, op,
@@ -1534,8 +1538,7 @@ rxm_send_common(struct rxm_ep *rxm_ep, struct rxm_conn *rxm_conn,
 				   context, data, flags, tag, op, data_len,
 				   rxm_ep_sar_calc_segs_cnt(rxm_ep, data_len));
 	} else {
-		iface = rxm_mr_desc_to_hmem_iface_dev(desc, count, &device);
-
+rndv_send:
 		ret = rxm_alloc_rndv_buf(rxm_ep, rxm_conn, context,
 					 (uint8_t) count, iov, desc,
 					 data_len, data, flags, tag, op,