prov/efa: Require FI_RX_CQ_DATA on devices without unsolicited write recv

When unsolicited write recv is not supported, RDMA write with immediate data
operations consume an RX buffer, requiring the FI_RX_CQ_DATA mode bit to be
set for FI_RMA.

cq_data_size shouldn't be used to indicate support for fi_writedata
because there are reasonable use cases where the cq_data_size is 0 and
the completion needs to be generated.

Signed-off-by: Jessie Yang <[email protected]>

Author: jiaxiyan

prov/efa/src/efa_user_info.c

@@ -478,18 +478,12 @@ int efa_user_info_alter_direct(int version, struct fi_info *info, const struct f
 		 * RDMA with immediate needs to consume a recv buffer
 		 * when unsolicited write recv is not supported. So
 		 * FI_RX_CQ_DATA is required when application requests
-		 * a non-zero cq data size in this situation.
+		 * FI_RMA.
 		 */
-		if (hints->domain_attr &&
-		    hints->domain_attr->cq_data_size > 0) {
-			EFA_INFO(FI_LOG_CORE,
-				 "FI_RX_CQ_DATA is required for FI_RMA + "
-				 "non-zero cq data when unsolicited write recv "
-				 "is not supported \n");
-			return -FI_ENODATA;
-		} else {
-			info->domain_attr->cq_data_size = 0;
-		}
+		EFA_INFO(FI_LOG_CORE,
+			 "FI_RX_CQ_DATA is required for FI_RMA when "
+			 "unsolicited write recv is not supported.\n");
+		return -FI_ENODATA;
 	}
 	/*
 	 * Handle user-provided hints and adapt the info object passed back up

prov/efa/test/efa_unit_test_info.c

@@ -944,8 +944,7 @@ void test_info_reuse_domain_via_name()
 	test_info_reuse_fabric_domain(setup_domain_name_hints, true, true);
 }
 static void test_info_direct_rma_common(bool mock_unsolicited_write_recv, bool set_rx_cq_data,
-					size_t request_cq_data_size, int expected_err,
-					size_t expected_cq_data_size, bool expect_rx_cq_data_mode)
+					int expected_err, size_t expected_cq_data_size, bool expect_rx_cq_data_mode)
 {
 	struct fi_info *hints, *info;
 	int err;
@@ -959,8 +958,6 @@ static void test_info_direct_rma_common(bool mock_unsolicited_write_recv, bool s
 	hints->caps |= FI_RMA;
 	if (set_rx_cq_data)
 		hints->mode |= FI_RX_CQ_DATA;
-	if (request_cq_data_size > 0)
-		hints->domain_attr->cq_data_size = request_cq_data_size;
 
 	err = fi_getinfo(FI_VERSION(1, 18), NULL, NULL, 0ULL, hints, &info);
 	fi_freeinfo(hints);
@@ -989,15 +986,15 @@ static void test_info_direct_rma_common(bool mock_unsolicited_write_recv, bool s
  */
 void test_info_direct_rma_when_no_unsolicited_write_recv_and_rx_cq_data()
 {
-	test_info_direct_rma_common(false, true, 0, 0, 4, true);
+	test_info_direct_rma_common(false, true, 0, 4, true);
 }
 
 /**
- * @brief Test that when FI_RX_CQ_DATA is not requested and unsolicited write recv is OFF, cq_data_size is 0
+ * @brief Test that when FI_RX_CQ_DATA is not requested and unsolicited write recv is OFF, fi_getinfo fails
  */
-void test_info_direct_rma_when_no_rx_cq_data_and_zero_cq_data_size()
+void test_info_direct_rma_when_no_unsolicited_write_recv_and_no_rx_cq_data()
 {
-	test_info_direct_rma_common(false, false, 0, 0, 0, false);
+	test_info_direct_rma_common(false, false, -FI_ENODATA, 0, false);
 }
 
 /**
@@ -1006,12 +1003,5 @@ void test_info_direct_rma_when_no_rx_cq_data_and_zero_cq_data_size()
 void test_info_direct_rma_when_unsolicited_write_recv_on_and_no_rx_cq_data()
 {
 	return;
-	test_info_direct_rma_common(true, false, 0, 0, 4, false);
-}
-/**
- * @brief Test that when user requests non-zero cq_data_size but unsolicited write recv is OFF, fi_getinfo fails
- */
-void test_info_direct_rma_when_no_unsolicited_write_recv_and_nonzero_cq_data_size_and_no_rx_cq_data()
-{
-	test_info_direct_rma_common(false, false, 4, -FI_ENODATA, 0, false);
+	test_info_direct_rma_common(true, false, 0, 4, false);
 }

prov/efa/test/efa_unit_tests.c

@@ -267,9 +267,8 @@ int main(void)
 		cmocka_unit_test_setup_teardown(test_efa_use_device_rdma_opt0, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_use_device_rdma_opt_old, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_info_direct_rma_when_no_unsolicited_write_recv_and_rx_cq_data, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
-		cmocka_unit_test_setup_teardown(test_info_direct_rma_when_no_rx_cq_data_and_zero_cq_data_size, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
+		cmocka_unit_test_setup_teardown(test_info_direct_rma_when_no_unsolicited_write_recv_and_no_rx_cq_data, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_info_direct_rma_when_unsolicited_write_recv_on_and_no_rx_cq_data, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
-		cmocka_unit_test_setup_teardown(test_info_direct_rma_when_no_unsolicited_write_recv_and_nonzero_cq_data_size_and_no_rx_cq_data, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		/* end efa_unit_test_info.c */
 
 		cmocka_unit_test_setup_teardown(test_efa_hmem_info_update_neuron, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),

prov/efa/test/efa_unit_tests.h

@@ -230,9 +230,8 @@ void test_efa_use_device_rdma_env1();
 void test_efa_use_device_rdma_env0();
 void test_efa_use_device_rdma_opt_old();
 void test_info_direct_rma_when_no_unsolicited_write_recv_and_rx_cq_data();
-void test_info_direct_rma_when_no_rx_cq_data_and_zero_cq_data_size();
+void test_info_direct_rma_when_no_unsolicited_write_recv_and_no_rx_cq_data();
 void test_info_direct_rma_when_unsolicited_write_recv_on_and_no_rx_cq_data();
-void test_info_direct_rma_when_no_unsolicited_write_recv_and_nonzero_cq_data_size_and_no_rx_cq_data();
 /* end efa_unit_test_info.c */
 
 void test_efa_srx_min_multi_recv_size();