From 49787d105302bd8cbb48fbb90fd00fa337c06da5 Mon Sep 17 00:00:00 2001 From: Sunita Nadampalli Date: Fri, 3 Oct 2025 02:49:11 +0000 Subject: [PATCH] prov/efa: make efa object allocations cache line aligned This is required to improve p99 latencies Signed-off-by: Sunita Nadampalli --- prov/efa/src/efa_av.c | 21 +++++++++++---------- prov/efa/src/efa_base_ep.c | 12 ++++++++---- prov/efa/src/efa_cq.c | 7 ++++--- prov/efa/src/efa_domain.c | 7 ++++--- prov/efa/src/rdm/efa_rdm_cq.c | 7 ++++--- prov/efa/src/rdm/efa_rdm_ep_fiops.c | 13 ++++++++----- 6 files changed, 39 insertions(+), 28 deletions(-) diff --git a/prov/efa/src/efa_av.c b/prov/efa/src/efa_av.c index eafcfeed462..6e106e03cd7 100644 --- a/prov/efa/src/efa_av.c +++ b/prov/efa/src/efa_av.c @@ -251,7 +251,7 @@ struct efa_ah *efa_ah_alloc(struct efa_domain *domain, const uint8_t *gid) return efa_ah; } - efa_ah = malloc(sizeof(struct efa_ah)); + ofi_memalign((void **)&efa_ah, EFA_MEM_ALIGNMENT, sizeof(struct efa_ah)); if (!efa_ah) { errno = FI_ENOMEM; EFA_WARN(FI_LOG_AV, "cannot allocate memory for efa_ah\n"); @@ -284,7 +284,7 @@ struct efa_ah *efa_ah_alloc(struct efa_domain *domain, const uint8_t *gid) err_destroy_ibv_ah: ibv_destroy_ah(efa_ah->ibv_ah); err_free_efa_ah: - free(efa_ah); + ofi_freealign(efa_ah); ofi_genlock_unlock(&domain->util_domain.lock); return NULL; } @@ -313,7 +313,7 @@ void efa_ah_release(struct efa_domain *domain, struct efa_ah *ah) err = ibv_destroy_ah(ah->ibv_ah); if (err) EFA_WARN(FI_LOG_AV, "ibv_destroy_ah failed! err=%d\n", err); - free(ah); + ofi_freealign(ah); } ofi_genlock_unlock(&domain->util_domain.lock); } @@ -469,7 +469,7 @@ int efa_av_reverse_av_add(struct efa_av *av, HASH_FIND(hh, *cur_reverse_av, &cur_key, sizeof(cur_key), cur_entry); if (!cur_entry) { - cur_entry = malloc(sizeof(*cur_entry)); + ofi_memalign((void **)&cur_entry, EFA_MEM_ALIGNMENT, sizeof(*cur_entry)); if (!cur_entry) { EFA_WARN(FI_LOG_AV, "Cannot allocate memory for cur_reverse_av entry\n"); return -FI_ENOMEM; @@ -487,7 +487,7 @@ int efa_av_reverse_av_add(struct efa_av *av, * and only RDM endpoint can reach here. hence the following assertion */ assert(av->domain->info_type == EFA_INFO_RDM); - prv_entry = malloc(sizeof(*prv_entry)); + ofi_memalign((void **)&prv_entry, EFA_MEM_ALIGNMENT, sizeof(*prv_entry)); if (!prv_entry) { EFA_WARN(FI_LOG_AV, "Cannot allocate memory for prv_reverse_av entry\n"); return -FI_ENOMEM; @@ -533,7 +533,7 @@ static void efa_av_reverse_av_remove(struct efa_cur_reverse_av **cur_reverse_av, cur_reverse_av_entry); if (cur_reverse_av_entry) { HASH_DEL(*cur_reverse_av, cur_reverse_av_entry); - free(cur_reverse_av_entry); + ofi_freealign(cur_reverse_av_entry); } else { memset(&prv_key, 0, sizeof(prv_key)); prv_key.ahn = conn->ah->ahn; @@ -543,7 +543,7 @@ static void efa_av_reverse_av_remove(struct efa_cur_reverse_av **cur_reverse_av, prv_reverse_av_entry); assert(prv_reverse_av_entry); HASH_DEL(*prv_reverse_av, prv_reverse_av_entry); - free(prv_reverse_av_entry); + ofi_freealign(prv_reverse_av_entry); } } @@ -1251,7 +1251,7 @@ static int efa_av_close(struct fid *fid) free(ep_addr_hashable); } - free(av); + ofi_freealign(av); return err; } @@ -1315,9 +1315,10 @@ int efa_av_open(struct fid_domain *domain_fid, struct fi_av_attr *attr, else attr->count = MAX(attr->count, EFA_MIN_AV_SIZE); - av = calloc(1, sizeof(*av)); + ofi_memalign((void **)&av, EFA_MEM_ALIGNMENT, sizeof(*av)); if (!av) return -FI_ENOMEM; + memset(av, 0x0, sizeof(*av)); if (attr->type == FI_AV_MAP) { EFA_INFO(FI_LOG_AV, "FI_AV_MAP is deprecated in Libfabric 2.x. Please use FI_AV_TABLE. " @@ -1396,6 +1397,6 @@ int efa_av_open(struct fid_domain *domain_fid, struct fi_av_attr *attr, "Unable to close util_av_implicit: %s\n", fi_strerror(-retv)); err: - free(av); + ofi_freealign(av); return ret; } diff --git a/prov/efa/src/efa_base_ep.c b/prov/efa/src/efa_base_ep.c index e3c95399c09..d476daa39a7 100644 --- a/prov/efa/src/efa_base_ep.c +++ b/prov/efa/src/efa_base_ep.c @@ -133,10 +133,10 @@ int efa_base_ep_destruct(struct efa_base_ep *base_ep) err = efa_base_ep_destruct_qp(base_ep); if (base_ep->efa_recv_wr_vec) - free(base_ep->efa_recv_wr_vec); + ofi_freealign(base_ep->efa_recv_wr_vec); if (base_ep->user_recv_wr_vec) - free(base_ep->user_recv_wr_vec); + ofi_freealign(base_ep->user_recv_wr_vec); return err; } @@ -483,16 +483,20 @@ int efa_base_ep_construct(struct efa_base_ep *base_ep, /* This is SRD qp's default behavior */ base_ep->rnr_retry = EFA_RNR_INFINITE_RETRY; - base_ep->efa_recv_wr_vec = calloc(sizeof(struct efa_recv_wr), EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV); + ofi_memalign((void **)&base_ep->efa_recv_wr_vec, EFA_MEM_ALIGNMENT, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV); if (!base_ep->efa_recv_wr_vec) { EFA_WARN(FI_LOG_EP_CTRL, "cannot alloc memory for base_ep->efa_recv_wr_vec!\n"); return -FI_ENOMEM; } - base_ep->user_recv_wr_vec = calloc(sizeof(struct efa_recv_wr), EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV); + memset(base_ep->efa_recv_wr_vec, 0x0, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV); + + ofi_memalign((void **)&base_ep->user_recv_wr_vec, EFA_MEM_ALIGNMENT, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV); if (!base_ep->user_recv_wr_vec) { EFA_WARN(FI_LOG_EP_CTRL, "cannot alloc memory for base_ep->user_recv_wr_vec!\n"); return -FI_ENOMEM; } + memset(base_ep->user_recv_wr_vec, 0x0, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV); + base_ep->recv_wr_index = 0; base_ep->efa_qp_enabled = false; base_ep->qp = NULL; diff --git a/prov/efa/src/efa_cq.c b/prov/efa/src/efa_cq.c index 1b8790757de..bd4aee0a80b 100644 --- a/prov/efa/src/efa_cq.c +++ b/prov/efa/src/efa_cq.c @@ -863,7 +863,7 @@ int efa_cq_close(fid_t fid) if (cq->err_buf) free(cq->err_buf); - free(cq); + ofi_freealign(cq); return 0; } @@ -936,11 +936,12 @@ int efa_cq_open(struct fid_domain *domain_fid, struct fi_cq_attr *attr, struct fi_cq_attr tmp_attr; int err, retv; - cq = calloc(1, sizeof(*cq)); + ofi_memalign((void **)&cq, EFA_MEM_ALIGNMENT, sizeof(*cq)); if (!cq) { EFA_WARN(FI_LOG_CQ, "Failed to allocate memory for CQ\n"); return -FI_ENOMEM; } + memset(cq, 0x0, sizeof(*cq)); cq->poll_ibv_cq = efa_cq_poll_ibv_cq; @@ -1068,6 +1069,6 @@ int efa_cq_open(struct fid_domain *domain_fid, struct fi_cq_attr *attr, EFA_WARN(FI_LOG_CQ, "Unable to close util cq: %s\n", fi_strerror(-retv)); err_free_cq: - free(cq); + ofi_freealign(cq); return err; } diff --git a/prov/efa/src/efa_domain.c b/prov/efa/src/efa_domain.c index 196562a125b..ecad0d302ea 100644 --- a/prov/efa/src/efa_domain.c +++ b/prov/efa/src/efa_domain.c @@ -191,9 +191,10 @@ int efa_domain_open(struct fid_fabric *fabric_fid, struct fi_info *info, int ret = 0, err; bool use_lock; - efa_domain = calloc(1, sizeof(struct efa_domain)); + ofi_memalign((void **)&efa_domain, EFA_MEM_ALIGNMENT, sizeof(struct efa_domain)); if (!efa_domain) return -FI_ENOMEM; + memset(efa_domain, 0x0, sizeof(struct efa_domain)); dlist_init(&efa_domain->list_entry); efa_domain->fabric = container_of(fabric_fid, struct efa_fabric, @@ -368,7 +369,7 @@ static int efa_domain_close(fid_t fid) if (ret) EFA_WARN(FI_LOG_DOMAIN, "ibv_destroy_ah failed during cleanup! err=%d\n", ret); HASH_DEL(efa_domain->ah_map, ah_entry); - free(ah_entry); + ofi_freealign(ah_entry); } } ofi_genlock_unlock(&efa_domain->util_domain.lock); @@ -398,7 +399,7 @@ static int efa_domain_close(fid_t fid) ofi_genlock_destroy(&efa_domain->srx_lock); free(efa_domain->qp_table); - free(efa_domain); + ofi_freealign(efa_domain); return 0; } diff --git a/prov/efa/src/rdm/efa_rdm_cq.c b/prov/efa/src/rdm/efa_rdm_cq.c index e1e5ac8371c..a398f815a71 100644 --- a/prov/efa/src/rdm/efa_rdm_cq.c +++ b/prov/efa/src/rdm/efa_rdm_cq.c @@ -61,7 +61,7 @@ int efa_rdm_cq_close(struct fid *fid) ret = ofi_cq_cleanup(&cq->efa_cq.util_cq); if (ret) return ret; - free(cq); + ofi_freealign(cq); return retv; } @@ -929,9 +929,10 @@ int efa_rdm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, if (attr->wait_obj != FI_WAIT_NONE) return -FI_ENOSYS; - cq = calloc(1, sizeof(*cq)); + ofi_memalign((void **)&cq, EFA_MEM_ALIGNMENT, sizeof(*cq)); if (!cq) return -FI_ENOMEM; + memset(cq, 0x0, sizeof(*cq)); efa_domain = container_of(domain, struct efa_domain, util_domain.domain_fid); @@ -987,6 +988,6 @@ int efa_rdm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, EFA_WARN(FI_LOG_CQ, "Unable to close util cq: %s\n", fi_strerror(-retv)); free: - free(cq); + ofi_freealign(cq); return ret; } diff --git a/prov/efa/src/rdm/efa_rdm_ep_fiops.c b/prov/efa/src/rdm/efa_rdm_ep_fiops.c index 7cc63bf804c..d15b1427bdc 100644 --- a/prov/efa/src/rdm/efa_rdm_ep_fiops.c +++ b/prov/efa/src/rdm/efa_rdm_ep_fiops.c @@ -466,10 +466,12 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info, int ret, retv, i; enum fi_hmem_iface iface; - efa_rdm_ep = calloc(1, sizeof(*efa_rdm_ep)); + ofi_memalign((void **)&efa_rdm_ep, EFA_MEM_ALIGNMENT, sizeof(*efa_rdm_ep)); if (!efa_rdm_ep) return -FI_ENOMEM; + memset(efa_rdm_ep, 0x0, sizeof(*efa_rdm_ep)); + efa_domain = container_of(domain, struct efa_domain, util_domain.domain_fid); @@ -582,12 +584,13 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info, efa_rdm_ep->write_in_order_aligned_128_bytes = false; efa_rdm_ep->homogeneous_peers = false; - efa_rdm_ep->pke_vec = calloc(sizeof(struct efa_rdm_pke *), EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV); + ofi_memalign((void **)&efa_rdm_ep->pke_vec, EFA_MEM_ALIGNMENT, sizeof(struct efa_rdm_pke *) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV); if (!efa_rdm_ep->pke_vec) { EFA_WARN(FI_LOG_EP_CTRL, "cannot alloc memory for efa_rdm_ep->pke_vec!\n"); ret = -FI_ENOMEM; goto err_close_shm_ep; } + memset(efa_rdm_ep->pke_vec, 0x0, sizeof(struct efa_rdm_pke *) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV); *ep = &efa_rdm_ep->base_ep.util_ep.ep_fid; (*ep)->msg = &efa_rdm_msg_ops; @@ -610,7 +613,7 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info, efa_base_ep_destruct(&efa_rdm_ep->base_ep); err_free_ep: if (efa_rdm_ep) - free(efa_rdm_ep); + ofi_freealign(efa_rdm_ep); return ret; } @@ -1040,11 +1043,11 @@ static int efa_rdm_ep_close(struct fid *fid) efa_rdm_ep_destroy_buffer_pools(efa_rdm_ep); if (efa_rdm_ep->pke_vec) - free(efa_rdm_ep->pke_vec); + ofi_freealign(efa_rdm_ep->pke_vec); ofi_genlock_unlock(&domain->srx_lock); - free(efa_rdm_ep); + ofi_freealign(efa_rdm_ep); return retv; }