prov/efa: Allocate cq err_buf on demand

efa_cq->err_buf is only needed by efa-direct cq which
can store the err data on the util cq bypass path.
Make it allocated on demand during the efa-direct
cq open to save memory.

Signed-off-by: Shi Jin <[email protected]>

Author: shijin-aws

prov/efa/src/efa_cq.c

@@ -860,6 +860,9 @@ int efa_cq_close(fid_t fid)
 	}
 #endif
 
+	if (cq->err_buf)
+		free(cq->err_buf);
+
 	free(cq);
 
 	return 0;
@@ -988,6 +991,14 @@ int efa_cq_open(struct fid_domain *domain_fid, struct fi_cq_attr *attr,
 
 	cq->wait_cond = attr->wait_cond;
 
+	/* This buffer is only used by efa-direct cq on the util cq bypass path */
+	cq->err_buf = malloc(EFA_ERROR_MSG_BUFFER_LENGTH);
+	if (!cq->err_buf) {
+		EFA_WARN(FI_LOG_CQ, "Failed to allocate memory for err_data buf in CQ\n");
+		err = -FI_ENOMEM;
+		goto err_free_util_cq;
+	}
+
 	err = efa_cq_open_ibv_cq(attr, efa_domain->device->ibv_ctx,
 				 &cq->ibv_cq,
 				 &efa_cq_init_attr);

prov/efa/src/efa_cq.h

@@ -39,8 +39,9 @@ struct efa_cq {
 	ofi_atomic32_t			nevents;
 	enum fi_wait_obj		wait_obj;
 	enum fi_cq_wait_cond	wait_cond;
+	/* Only used by efa-direct cq on util cq bypass path */
 	void (*read_entry)(struct efa_ibv_cq *ibv_cq, void *buf, int opcode);
-	char err_buf[EFA_ERROR_MSG_BUFFER_LENGTH];
+	char *err_buf;
 };
 
 extern struct fi_ops_cq efa_cq_ops;