[v2.3.x] prov/efa: Use IBV_QUERY_QP_DATA_IN_ORDER_DEVICE_ONLY flag when available

Add configure check and use the flag in ibv_query_qp_data_in_order()
calls when available. This flag queries device-only data ordering
capabilities without considering CPU architecture, needed for GPU
polling scenarios.

Reviewed-by: Daniel Kranzdorf <[email protected]>
Signed-off-by: Michael Margolin <[email protected]>
(cherry picked from commit 146b839)

Author: mrgolin

prov/efa/configure.m4

@@ -81,6 +81,7 @@ AC_DEFUN([FI_EFA_CONFIGURE],[
 	have_caps_cq_with_ext_mem_dmabuf=0
 	have_ibv_is_fork_initialized=0
 	efa_support_data_in_order_aligned_128_byte=0
+	have_ibv_query_qp_data_in_order_device_only=0
 	efadv_support_extended_cq=0
 	have_efa_dmabuf_mr=0
 	have_efadv_query_mr=0
@@ -130,6 +131,11 @@ AC_DEFUN([FI_EFA_CONFIGURE],[
 			[efa_support_data_in_order_aligned_128_byte=0],
 			[[#include <infiniband/verbs.h>]])
 
+		AC_CHECK_DECL([IBV_QUERY_QP_DATA_IN_ORDER_DEVICE_ONLY],
+			[have_ibv_query_qp_data_in_order_device_only=1],
+			[have_ibv_query_qp_data_in_order_device_only=0],
+			[[#include <infiniband/verbs.h>]])
+
 		AC_CHECK_DECL([ibv_reg_dmabuf_mr],
 			[have_efa_dmabuf_mr=1],
 			[have_efa_dmabuf_mr=0],
@@ -236,6 +242,9 @@ AC_DEFUN([FI_EFA_CONFIGURE],[
 	AC_DEFINE_UNQUOTED([HAVE_EFA_DATA_IN_ORDER_ALIGNED_128_BYTES],
 		[$efa_support_data_in_order_aligned_128_byte],
 		[Indicates if EFA supports 128 bytes in-order in writing.])
+	AC_DEFINE_UNQUOTED([HAVE_IBV_QUERY_QP_DATA_IN_ORDER_DEVICE_ONLY],
+		[$have_ibv_query_qp_data_in_order_device_only],
+		[Indicates if IBV_QUERY_QP_DATA_IN_ORDER_DEVICE_ONLY flag is available.])
 	AC_DEFINE_UNQUOTED([HAVE_EFA_DMABUF_MR],
 		[$have_efa_dmabuf_mr],
 		[Indicates if ibv_reg_dmabuf_mr verbs is available])

prov/efa/src/efa_base_ep.c

@@ -542,9 +542,13 @@ int efa_base_ep_getname(fid_t fid, void *addr, size_t *addrlen)
 bool efa_qp_support_op_in_order_aligned_128_bytes(struct efa_qp *qp, enum ibv_wr_opcode op)
 {
 	int caps;
+	uint32_t flags = IBV_QUERY_QP_DATA_IN_ORDER_RETURN_CAPS;
 
-	caps = ibv_query_qp_data_in_order(qp->ibv_qp, op,
-					  IBV_QUERY_QP_DATA_IN_ORDER_RETURN_CAPS);
+#if HAVE_IBV_QUERY_QP_DATA_IN_ORDER_DEVICE_ONLY
+	flags |= IBV_QUERY_QP_DATA_IN_ORDER_DEVICE_ONLY;
+#endif
+
+	caps = ibv_query_qp_data_in_order(qp->ibv_qp, op, flags);
 
 	return !!(caps & IBV_QUERY_QP_DATA_IN_ORDER_ALIGNED_128_BYTES);
 }