Skip to content

Commit 49787d1

Browse files
snadampalsnadampal
committed
prov/efa: make efa object allocations cache line aligned
This is required to improve p99 latencies Signed-off-by: Sunita Nadampalli <[email protected]>
1 parent 7d60749 commit 49787d1

File tree

6 files changed

+39
-28
lines changed

6 files changed

+39
-28
lines changed

‎prov/efa/src/efa_av.c‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ struct efa_ah *efa_ah_alloc(struct efa_domain *domain, const uint8_t *gid)
251251
return efa_ah;
252252
}
253253

254-
efa_ah = malloc(sizeof(struct efa_ah));
254+
ofi_memalign((void **)&efa_ah, EFA_MEM_ALIGNMENT, sizeof(struct efa_ah));
255255
if (!efa_ah) {
256256
errno = FI_ENOMEM;
257257
EFA_WARN(FI_LOG_AV, "cannot allocate memory for efa_ah\n");
@@ -284,7 +284,7 @@ struct efa_ah *efa_ah_alloc(struct efa_domain *domain, const uint8_t *gid)
284284
err_destroy_ibv_ah:
285285
ibv_destroy_ah(efa_ah->ibv_ah);
286286
err_free_efa_ah:
287-
free(efa_ah);
287+
ofi_freealign(efa_ah);
288288
ofi_genlock_unlock(&domain->util_domain.lock);
289289
return NULL;
290290
}
@@ -313,7 +313,7 @@ void efa_ah_release(struct efa_domain *domain, struct efa_ah *ah)
313313
err = ibv_destroy_ah(ah->ibv_ah);
314314
if (err)
315315
EFA_WARN(FI_LOG_AV, "ibv_destroy_ah failed! err=%d\n", err);
316-
free(ah);
316+
ofi_freealign(ah);
317317
}
318318
ofi_genlock_unlock(&domain->util_domain.lock);
319319
}
@@ -469,7 +469,7 @@ int efa_av_reverse_av_add(struct efa_av *av,
469469

470470
HASH_FIND(hh, *cur_reverse_av, &cur_key, sizeof(cur_key), cur_entry);
471471
if (!cur_entry) {
472-
cur_entry = malloc(sizeof(*cur_entry));
472+
ofi_memalign((void **)&cur_entry, EFA_MEM_ALIGNMENT, sizeof(*cur_entry));
473473
if (!cur_entry) {
474474
EFA_WARN(FI_LOG_AV, "Cannot allocate memory for cur_reverse_av entry\n");
475475
return -FI_ENOMEM;
@@ -487,7 +487,7 @@ int efa_av_reverse_av_add(struct efa_av *av,
487487
* and only RDM endpoint can reach here. hence the following assertion
488488
*/
489489
assert(av->domain->info_type == EFA_INFO_RDM);
490-
prv_entry = malloc(sizeof(*prv_entry));
490+
ofi_memalign((void **)&prv_entry, EFA_MEM_ALIGNMENT, sizeof(*prv_entry));
491491
if (!prv_entry) {
492492
EFA_WARN(FI_LOG_AV, "Cannot allocate memory for prv_reverse_av entry\n");
493493
return -FI_ENOMEM;
@@ -533,7 +533,7 @@ static void efa_av_reverse_av_remove(struct efa_cur_reverse_av **cur_reverse_av,
533533
cur_reverse_av_entry);
534534
if (cur_reverse_av_entry) {
535535
HASH_DEL(*cur_reverse_av, cur_reverse_av_entry);
536-
free(cur_reverse_av_entry);
536+
ofi_freealign(cur_reverse_av_entry);
537537
} else {
538538
memset(&prv_key, 0, sizeof(prv_key));
539539
prv_key.ahn = conn->ah->ahn;
@@ -543,7 +543,7 @@ static void efa_av_reverse_av_remove(struct efa_cur_reverse_av **cur_reverse_av,
543543
prv_reverse_av_entry);
544544
assert(prv_reverse_av_entry);
545545
HASH_DEL(*prv_reverse_av, prv_reverse_av_entry);
546-
free(prv_reverse_av_entry);
546+
ofi_freealign(prv_reverse_av_entry);
547547
}
548548
}
549549

@@ -1251,7 +1251,7 @@ static int efa_av_close(struct fid *fid)
12511251
free(ep_addr_hashable);
12521252
}
12531253

1254-
free(av);
1254+
ofi_freealign(av);
12551255
return err;
12561256
}
12571257

@@ -1315,9 +1315,10 @@ int efa_av_open(struct fid_domain *domain_fid, struct fi_av_attr *attr,
13151315
else
13161316
attr->count = MAX(attr->count, EFA_MIN_AV_SIZE);
13171317

1318-
av = calloc(1, sizeof(*av));
1318+
ofi_memalign((void **)&av, EFA_MEM_ALIGNMENT, sizeof(*av));
13191319
if (!av)
13201320
return -FI_ENOMEM;
1321+
memset(av, 0x0, sizeof(*av));
13211322

13221323
if (attr->type == FI_AV_MAP) {
13231324
EFA_INFO(FI_LOG_AV, "FI_AV_MAP is deprecated in Libfabric 2.x. Please use FI_AV_TABLE. "
@@ -1396,6 +1397,6 @@ int efa_av_open(struct fid_domain *domain_fid, struct fi_av_attr *attr,
13961397
"Unable to close util_av_implicit: %s\n", fi_strerror(-retv));
13971398

13981399
err:
1399-
free(av);
1400+
ofi_freealign(av);
14001401
return ret;
14011402
}

‎prov/efa/src/efa_base_ep.c‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -133,10 +133,10 @@ int efa_base_ep_destruct(struct efa_base_ep *base_ep)
133133
err = efa_base_ep_destruct_qp(base_ep);
134134

135135
if (base_ep->efa_recv_wr_vec)
136-
free(base_ep->efa_recv_wr_vec);
136+
ofi_freealign(base_ep->efa_recv_wr_vec);
137137

138138
if (base_ep->user_recv_wr_vec)
139-
free(base_ep->user_recv_wr_vec);
139+
ofi_freealign(base_ep->user_recv_wr_vec);
140140

141141
return err;
142142
}
@@ -483,16 +483,20 @@ int efa_base_ep_construct(struct efa_base_ep *base_ep,
483483
/* This is SRD qp's default behavior */
484484
base_ep->rnr_retry = EFA_RNR_INFINITE_RETRY;
485485

486-
base_ep->efa_recv_wr_vec = calloc(sizeof(struct efa_recv_wr), EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
486+
ofi_memalign((void **)&base_ep->efa_recv_wr_vec, EFA_MEM_ALIGNMENT, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
487487
if (!base_ep->efa_recv_wr_vec) {
488488
EFA_WARN(FI_LOG_EP_CTRL, "cannot alloc memory for base_ep->efa_recv_wr_vec!\n");
489489
return -FI_ENOMEM;
490490
}
491-
base_ep->user_recv_wr_vec = calloc(sizeof(struct efa_recv_wr), EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
491+
memset(base_ep->efa_recv_wr_vec, 0x0, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
492+
493+
ofi_memalign((void **)&base_ep->user_recv_wr_vec, EFA_MEM_ALIGNMENT, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
492494
if (!base_ep->user_recv_wr_vec) {
493495
EFA_WARN(FI_LOG_EP_CTRL, "cannot alloc memory for base_ep->user_recv_wr_vec!\n");
494496
return -FI_ENOMEM;
495497
}
498+
memset(base_ep->user_recv_wr_vec, 0x0, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
499+
496500
base_ep->recv_wr_index = 0;
497501
base_ep->efa_qp_enabled = false;
498502
base_ep->qp = NULL;

‎prov/efa/src/efa_cq.c‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -863,7 +863,7 @@ int efa_cq_close(fid_t fid)
863863
if (cq->err_buf)
864864
free(cq->err_buf);
865865

866-
free(cq);
866+
ofi_freealign(cq);
867867

868868
return 0;
869869
}
@@ -936,11 +936,12 @@ int efa_cq_open(struct fid_domain *domain_fid, struct fi_cq_attr *attr,
936936
struct fi_cq_attr tmp_attr;
937937
int err, retv;
938938

939-
cq = calloc(1, sizeof(*cq));
939+
ofi_memalign((void **)&cq, EFA_MEM_ALIGNMENT, sizeof(*cq));
940940
if (!cq) {
941941
EFA_WARN(FI_LOG_CQ, "Failed to allocate memory for CQ\n");
942942
return -FI_ENOMEM;
943943
}
944+
memset(cq, 0x0, sizeof(*cq));
944945

945946
cq->poll_ibv_cq = efa_cq_poll_ibv_cq;
946947

@@ -1068,6 +1069,6 @@ int efa_cq_open(struct fid_domain *domain_fid, struct fi_cq_attr *attr,
10681069
EFA_WARN(FI_LOG_CQ, "Unable to close util cq: %s\n",
10691070
fi_strerror(-retv));
10701071
err_free_cq:
1071-
free(cq);
1072+
ofi_freealign(cq);
10721073
return err;
10731074
}

‎prov/efa/src/efa_domain.c‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -191,9 +191,10 @@ int efa_domain_open(struct fid_fabric *fabric_fid, struct fi_info *info,
191191
int ret = 0, err;
192192
bool use_lock;
193193

194-
efa_domain = calloc(1, sizeof(struct efa_domain));
194+
ofi_memalign((void **)&efa_domain, EFA_MEM_ALIGNMENT, sizeof(struct efa_domain));
195195
if (!efa_domain)
196196
return -FI_ENOMEM;
197+
memset(efa_domain, 0x0, sizeof(struct efa_domain));
197198

198199
dlist_init(&efa_domain->list_entry);
199200
efa_domain->fabric = container_of(fabric_fid, struct efa_fabric,
@@ -368,7 +369,7 @@ static int efa_domain_close(fid_t fid)
368369
if (ret)
369370
EFA_WARN(FI_LOG_DOMAIN, "ibv_destroy_ah failed during cleanup! err=%d\n", ret);
370371
HASH_DEL(efa_domain->ah_map, ah_entry);
371-
free(ah_entry);
372+
ofi_freealign(ah_entry);
372373
}
373374
}
374375
ofi_genlock_unlock(&efa_domain->util_domain.lock);
@@ -398,7 +399,7 @@ static int efa_domain_close(fid_t fid)
398399

399400
ofi_genlock_destroy(&efa_domain->srx_lock);
400401
free(efa_domain->qp_table);
401-
free(efa_domain);
402+
ofi_freealign(efa_domain);
402403
return 0;
403404
}
404405

‎prov/efa/src/rdm/efa_rdm_cq.c‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ int efa_rdm_cq_close(struct fid *fid)
6161
ret = ofi_cq_cleanup(&cq->efa_cq.util_cq);
6262
if (ret)
6363
return ret;
64-
free(cq);
64+
ofi_freealign(cq);
6565
return retv;
6666
}
6767

@@ -929,9 +929,10 @@ int efa_rdm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
929929
if (attr->wait_obj != FI_WAIT_NONE)
930930
return -FI_ENOSYS;
931931

932-
cq = calloc(1, sizeof(*cq));
932+
ofi_memalign((void **)&cq, EFA_MEM_ALIGNMENT, sizeof(*cq));
933933
if (!cq)
934934
return -FI_ENOMEM;
935+
memset(cq, 0x0, sizeof(*cq));
935936

936937
efa_domain = container_of(domain, struct efa_domain,
937938
util_domain.domain_fid);
@@ -987,6 +988,6 @@ int efa_rdm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
987988
EFA_WARN(FI_LOG_CQ, "Unable to close util cq: %s\n",
988989
fi_strerror(-retv));
989990
free:
990-
free(cq);
991+
ofi_freealign(cq);
991992
return ret;
992993
}

‎prov/efa/src/rdm/efa_rdm_ep_fiops.c‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -466,10 +466,12 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info,
466466
int ret, retv, i;
467467
enum fi_hmem_iface iface;
468468

469-
efa_rdm_ep = calloc(1, sizeof(*efa_rdm_ep));
469+
ofi_memalign((void **)&efa_rdm_ep, EFA_MEM_ALIGNMENT, sizeof(*efa_rdm_ep));
470470
if (!efa_rdm_ep)
471471
return -FI_ENOMEM;
472472

473+
memset(efa_rdm_ep, 0x0, sizeof(*efa_rdm_ep));
474+
473475
efa_domain = container_of(domain, struct efa_domain,
474476
util_domain.domain_fid);
475477

@@ -582,12 +584,13 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info,
582584
efa_rdm_ep->write_in_order_aligned_128_bytes = false;
583585
efa_rdm_ep->homogeneous_peers = false;
584586

585-
efa_rdm_ep->pke_vec = calloc(sizeof(struct efa_rdm_pke *), EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
587+
ofi_memalign((void **)&efa_rdm_ep->pke_vec, EFA_MEM_ALIGNMENT, sizeof(struct efa_rdm_pke *) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
586588
if (!efa_rdm_ep->pke_vec) {
587589
EFA_WARN(FI_LOG_EP_CTRL, "cannot alloc memory for efa_rdm_ep->pke_vec!\n");
588590
ret = -FI_ENOMEM;
589591
goto err_close_shm_ep;
590592
}
593+
memset(efa_rdm_ep->pke_vec, 0x0, sizeof(struct efa_rdm_pke *) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
591594

592595
*ep = &efa_rdm_ep->base_ep.util_ep.ep_fid;
593596
(*ep)->msg = &efa_rdm_msg_ops;
@@ -610,7 +613,7 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info,
610613
efa_base_ep_destruct(&efa_rdm_ep->base_ep);
611614
err_free_ep:
612615
if (efa_rdm_ep)
613-
free(efa_rdm_ep);
616+
ofi_freealign(efa_rdm_ep);
614617
return ret;
615618
}
616619

@@ -1040,11 +1043,11 @@ static int efa_rdm_ep_close(struct fid *fid)
10401043
efa_rdm_ep_destroy_buffer_pools(efa_rdm_ep);
10411044

10421045
if (efa_rdm_ep->pke_vec)
1043-
free(efa_rdm_ep->pke_vec);
1046+
ofi_freealign(efa_rdm_ep->pke_vec);
10441047

10451048
ofi_genlock_unlock(&domain->srx_lock);
10461049

1047-
free(efa_rdm_ep);
1050+
ofi_freealign(efa_rdm_ep);
10481051
return retv;
10491052
}
10501053

0 commit comments

Comments
 (0)