prov/efa: make the efa object allocations cache line aligned

This is required to improve p99 latencies

Author: snadampal

prov/efa/src/efa_av.c

@@ -251,7 +251,7 @@ struct efa_ah *efa_ah_alloc(struct efa_domain *domain, const uint8_t *gid)
 		return efa_ah;
 	}
 
-	efa_ah = malloc(sizeof(struct efa_ah));
+	ofi_memalign((void **)&efa_ah, EFA_MEM_ALIGNMENT, sizeof(struct efa_ah));
 	if (!efa_ah) {
 		errno = FI_ENOMEM;
 		EFA_WARN(FI_LOG_AV, "cannot allocate memory for efa_ah\n");
@@ -469,7 +469,7 @@ int efa_av_reverse_av_add(struct efa_av *av,
 
 	HASH_FIND(hh, *cur_reverse_av, &cur_key, sizeof(cur_key), cur_entry);
 	if (!cur_entry) {
-		cur_entry = malloc(sizeof(*cur_entry));
+		ofi_memalign((void **)&cur_entry, EFA_MEM_ALIGNMENT, sizeof(*cur_entry));
 		if (!cur_entry) {
 			EFA_WARN(FI_LOG_AV, "Cannot allocate memory for cur_reverse_av entry\n");
 			return -FI_ENOMEM;
@@ -487,7 +487,7 @@ int efa_av_reverse_av_add(struct efa_av *av,
 	 * and only RDM endpoint can reach here. hence the following assertion
 	 */
 	assert(av->domain->info_type == EFA_INFO_RDM);
-	prv_entry = malloc(sizeof(*prv_entry));
+	ofi_memalign((void **)&prv_entry, 64, sizeof(*prv_entry));
 	if (!prv_entry) {
 		EFA_WARN(FI_LOG_AV, "Cannot allocate memory for prv_reverse_av entry\n");
 		return -FI_ENOMEM;
@@ -1315,9 +1315,10 @@ int efa_av_open(struct fid_domain *domain_fid, struct fi_av_attr *attr,
 	else
 		attr->count = MAX(attr->count, EFA_MIN_AV_SIZE);
 
-	av = calloc(1, sizeof(*av));
+	ofi_memalign((void **)&av, EFA_MEM_ALIGNMENT, sizeof(*av));
 	if (!av)
 		return -FI_ENOMEM;
+	memset(av, 0x0, sizeof(*av));
 
 	if (attr->type == FI_AV_MAP) {
 		EFA_INFO(FI_LOG_AV, "FI_AV_MAP is deprecated in Libfabric 2.x. Please use FI_AV_TABLE. "

prov/efa/src/efa_base_ep.c

@@ -483,16 +483,20 @@ int efa_base_ep_construct(struct efa_base_ep *base_ep,
 	/* This is SRD qp's default behavior */
 	base_ep->rnr_retry = EFA_RNR_INFINITE_RETRY;
 
-	base_ep->efa_recv_wr_vec = calloc(sizeof(struct efa_recv_wr), EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
+	ofi_memalign((void **)&base_ep->efa_recv_wr_vec, EFA_MEM_ALIGNMENT, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
 	if (!base_ep->efa_recv_wr_vec) {
 		EFA_WARN(FI_LOG_EP_CTRL, "cannot alloc memory for base_ep->efa_recv_wr_vec!\n");
 		return -FI_ENOMEM;
 	}
-	base_ep->user_recv_wr_vec = calloc(sizeof(struct efa_recv_wr), EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
+	memset(base_ep->efa_recv_wr_vec, 0x0, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
+
+	ofi_memalign((void **)&base_ep->user_recv_wr_vec, EFA_MEM_ALIGNMENT, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
 	if (!base_ep->user_recv_wr_vec) {
 		EFA_WARN(FI_LOG_EP_CTRL, "cannot alloc memory for base_ep->user_recv_wr_vec!\n");
 		return -FI_ENOMEM;
 	}
+	memset(base_ep->user_recv_wr_vec, 0x0, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
+
 	base_ep->recv_wr_index = 0;
 	base_ep->efa_qp_enabled = false;
 	base_ep->qp = NULL;

prov/efa/src/efa_cq.c

@@ -936,11 +936,12 @@ int efa_cq_open(struct fid_domain *domain_fid, struct fi_cq_attr *attr,
 	struct fi_cq_attr tmp_attr;
 	int err, retv;
 
-	cq = calloc(1, sizeof(*cq));
+	ofi_memalign((void **)&cq, EFA_MEM_ALIGNMENT, sizeof(*cq));
 	if (!cq) {
 		EFA_WARN(FI_LOG_CQ, "Failed to allocate memory for CQ\n");
 		return -FI_ENOMEM;
 	}
+	memset(cq, 0x0, sizeof(*cq));
 
 	cq->poll_ibv_cq = efa_cq_poll_ibv_cq;
 

prov/efa/src/rdm/efa_rdm_cq.c

@@ -929,9 +929,10 @@ int efa_rdm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
 	if (attr->wait_obj != FI_WAIT_NONE)
 		return -FI_ENOSYS;
 
-	cq = calloc(1, sizeof(*cq));
+	ofi_memalign((void **)&cq, EFA_MEM_ALIGNMENT, sizeof(*cq));
 	if (!cq)
 		return -FI_ENOMEM;
+	memset(cq, 0x0, sizeof(*cq));
 
 	efa_domain = container_of(domain, struct efa_domain,
 				  util_domain.domain_fid);

prov/efa/src/rdm/efa_rdm_ep_fiops.c

@@ -466,10 +466,12 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info,
 	int ret, retv, i;
 	enum fi_hmem_iface iface;
 
-	efa_rdm_ep = calloc(1, sizeof(*efa_rdm_ep));
+	ofi_memalign((void **)&efa_rdm_ep, EFA_MEM_ALIGNMENT, sizeof(*efa_rdm_ep));
 	if (!efa_rdm_ep)
 		return -FI_ENOMEM;
 
+	memset(efa_rdm_ep, 0x0, sizeof(*efa_rdm_ep));
+
 	efa_domain = container_of(domain, struct efa_domain,
 				  util_domain.domain_fid);
 
@@ -582,12 +584,13 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info,
 	efa_rdm_ep->write_in_order_aligned_128_bytes = false;
 	efa_rdm_ep->homogeneous_peers = false;
 
-	efa_rdm_ep->pke_vec = calloc(sizeof(struct efa_rdm_pke *), EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
+	ofi_memalign((void **)&efa_rdm_ep->pke_vec, EFA_MEM_ALIGNMENT, sizeof(struct efa_rdm_pke *) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
 	if (!efa_rdm_ep->pke_vec) {
 		EFA_WARN(FI_LOG_EP_CTRL, "cannot alloc memory for efa_rdm_ep->pke_vec!\n");
 		ret = -FI_ENOMEM;
 		goto err_close_shm_ep;
 	}
+	memset(efa_rdm_ep->pke_vec, 0x0, sizeof(struct efa_rdm_pke *) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
 
 	*ep = &efa_rdm_ep->base_ep.util_ep.ep_fid;
 	(*ep)->msg = &efa_rdm_msg_ops;

prov/efa/src/rdm/efa_rdm_ep_utils.c

@@ -1021,7 +1021,7 @@ void efa_rdm_ep_post_internal_rx_pkts(struct efa_rdm_ep *ep)
  */
 size_t efa_rdm_ep_get_memory_alignment(struct efa_rdm_ep *ep, enum fi_hmem_iface iface)
 {
-	size_t memory_alignment = EFA_RDM_DEFAULT_MEMORY_ALIGNMENT;
+	size_t memory_alignment = EFA_MEM_ALIGNMENT;
 
 	if (ep->sendrecv_in_order_aligned_128_bytes) {
 		memory_alignment = EFA_RDM_IN_ORDER_ALIGNMENT;