Skip to content

Commit f769f68

Browse files
hzhouhzhou
committed
ch4/ofi: use pipeline_pool for pack buffer
When we use malloc for pack buffer and when it is used for copying from and to GPU, we potentially can incur expensive registration cost every time. This is the case with ZE. Use pipeline_pool allows reusage of pack buffers.
1 parent 059ea00 commit f769f68

File tree

7 files changed

+35
-30
lines changed

7 files changed

+35
-30
lines changed

‎src/mpid/ch4/netmod/ofi/ofi_events.c‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -568,9 +568,8 @@ int MPIDI_OFI_handle_cq_error(int vci, int nic, ssize_t ret)
568568
MPIR_STATUS_SET_CANCEL_BIT(req->status, TRUE);
569569
MPIR_STATUS_SET_COUNT(req->status, 0);
570570
MPIR_Datatype_release_if_not_builtin(MPIDI_OFI_REQUEST(req, datatype));
571-
if ((event_id == MPIDI_OFI_EVENT_RECV_PACK) &&
572-
MPIDI_OFI_REQUEST(req, noncontig.pack.pack_buffer)) {
573-
MPL_free(MPIDI_OFI_REQUEST(req, noncontig.pack.pack_buffer));
571+
if (event_id == MPIDI_OFI_EVENT_RECV_PACK) {
572+
MPIDI_OFI_free_pack_buffer(req);
574573
} else if (event_id == MPIDI_OFI_EVENT_RECV_NOPACK) {
575574
MPL_free(MPIDI_OFI_REQUEST(req, noncontig.nopack.iovs));
576575
}

‎src/mpid/ch4/netmod/ofi/ofi_events.h‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,8 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_send_event(int vci,
4646
MPIR_FUNC_ENTER;
4747

4848
/* free the packing buffers and datatype */
49-
if ((event_id == MPIDI_OFI_EVENT_SEND_PACK) &&
50-
(MPIDI_OFI_REQUEST(sreq, noncontig.pack.pack_buffer))) {
51-
MPL_free(MPIDI_OFI_REQUEST(sreq, noncontig.pack.pack_buffer));
49+
if (event_id == MPIDI_OFI_EVENT_SEND_PACK) {
50+
MPIDI_OFI_free_pack_buffer(sreq);
5251
} else if (MPIDI_OFI_ENABLE_PT2PT_NOPACK && (event_id == MPIDI_OFI_EVENT_SEND_NOPACK)) {
5352
MPL_free(MPIDI_OFI_REQUEST(sreq, noncontig.nopack.iovs));
5453
}
@@ -97,7 +96,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_recv_complete(MPIR_Request * rreq, int ev
9796
if (mpi_errno) {
9897
MPIR_ERR_SET(rreq->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch");
9998
}
100-
MPL_free(MPIDI_OFI_REQUEST(rreq, noncontig.pack.pack_buffer));
99+
MPIDI_OFI_free_pack_buffer(rreq);
101100
} else if (event_id == MPIDI_OFI_EVENT_RECV_NOPACK) {
102101
#ifdef HAVE_ERROR_CHECKING
103102
MPI_Count elements;

‎src/mpid/ch4/netmod/ofi/ofi_impl.h‎

Lines changed: 26 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -833,27 +833,36 @@ MPL_STATIC_INLINE_PREFIX void MPIDI_OFI_gpu_rma_register(const void *buffer, siz
833833
#undef CQ_D_HEAD
834834
#undef CQ_D_TAIL
835835

836-
MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_gpu_malloc_pack_buffer(void **ptr, size_t pack_size)
837-
{
838-
if (MPIDI_OFI_ENABLE_HMEM) {
839-
return MPL_gpu_malloc_host(ptr, pack_size);
840-
} else {
841-
#ifdef MPL_DEFINE_ALIGNED_ALLOC
842-
*ptr = MPL_aligned_alloc(256, pack_size, MPL_MEM_BUFFER);
843-
#else
844-
*ptr = MPL_malloc(pack_size, MPL_MEM_BUFFER);
845-
#endif
846-
return 0;
836+
MPL_STATIC_INLINE_PREFIX void *MPIDI_OFI_malloc_pack_buffer(MPIR_Request * req, MPI_Aint pack_size)
837+
{
838+
void *pack_buf;
839+
bool is_genq;
840+
if (pack_size <= MPIR_CVAR_CH4_OFI_PIPELINE_CHUNK_SZ) {
841+
int vci = MPIR_REQUEST_POOL_FROM_HANDLE(req->handle);
842+
MPIDU_genq_private_pool_alloc_cell(MPIDI_OFI_global.per_vci[vci].pipeline_pool, &pack_buf);
843+
is_genq = true;
844+
}
845+
if (!pack_buf) {
846+
pack_buf = MPL_aligned_alloc(64, pack_size, MPL_MEM_OTHER);
847+
is_genq = false;
847848
}
849+
if (pack_buf) {
850+
MPIDI_OFI_REQUEST(req, noncontig.pack.pack_buffer) = pack_buf;
851+
MPIDI_OFI_REQUEST(req, noncontig.pack.is_genq) = is_genq;
852+
}
853+
return pack_buf;
848854
}
849855

850-
MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_gpu_free_pack_buffer(void *ptr)
856+
MPL_STATIC_INLINE_PREFIX void MPIDI_OFI_free_pack_buffer(MPIR_Request * req)
851857
{
852-
if (MPIDI_OFI_ENABLE_HMEM) {
853-
return MPL_gpu_free_host(ptr);
854-
} else {
855-
MPL_free(ptr);
856-
return 0;
858+
if (MPIDI_OFI_REQUEST(req, noncontig.pack.pack_buffer)) {
859+
if (MPIDI_OFI_REQUEST(req, noncontig.pack.is_genq)) {
860+
int vci = MPIR_REQUEST_POOL_FROM_HANDLE(req->handle);
861+
MPIDU_genq_private_pool_free_cell(MPIDI_OFI_global.per_vci[vci].pipeline_pool,
862+
MPIDI_OFI_REQUEST(req, noncontig.pack.pack_buffer));
863+
} else {
864+
MPL_free(MPIDI_OFI_REQUEST(req, noncontig.pack.pack_buffer));
865+
}
857866
}
858867
}
859868

‎src/mpid/ch4/netmod/ofi/ofi_pre.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,7 @@ typedef struct {
302302
enum MPIDI_OFI_req_kind kind;
303303
union {
304304
struct {
305+
bool is_genq;
305306
char *pack_buffer;
306307
} pack;
307308
struct {

‎src/mpid/ch4/netmod/ofi/ofi_recv.h‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -224,11 +224,9 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_do_irecv(void *buf,
224224

225225
/* Unpack */
226226
MPIDI_OFI_REQUEST(rreq, event_id) = MPIDI_OFI_EVENT_RECV_PACK;
227-
MPIDI_OFI_REQUEST(rreq, noncontig.pack.pack_buffer) =
228-
MPL_aligned_alloc(64, data_sz, MPL_MEM_OTHER);
227+
recv_buf = MPIDI_OFI_malloc_pack_buffer(rreq, data_sz);
229228
MPIR_ERR_CHKANDJUMP1(MPIDI_OFI_REQUEST(rreq, noncontig.pack.pack_buffer) == NULL, mpi_errno,
230229
MPI_ERR_OTHER, "**nomem", "**nomem %s", "Recv Pack Buffer alloc");
231-
recv_buf = MPIDI_OFI_REQUEST(rreq, noncontig.pack.pack_buffer);
232230
} else {
233231
MPIDI_OFI_REQUEST(rreq, noncontig.pack.pack_buffer) = NULL;
234232
}

‎src/mpid/ch4/netmod/ofi/ofi_rndv.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ int MPIDI_OFI_recv_rndv_event(int vci, struct fi_cq_tagged_entry *wc, MPIR_Reque
9191
/* if we were expecting an eager send, free the unneeded pack_buffer or iovs array */
9292
switch (MPIDI_OFI_REQUEST(rreq, event_id)) {
9393
case MPIDI_OFI_EVENT_RECV_PACK:
94-
MPL_free(MPIDI_OFI_REQUEST(rreq, noncontig.pack.pack_buffer));
94+
MPIDI_OFI_free_pack_buffer(rreq);
9595
break;
9696
case MPIDI_OFI_EVENT_RECV_NOPACK:
9797
MPL_free(MPIDI_OFI_REQUEST(rreq, noncontig.nopack.iovs));

‎src/mpid/ch4/netmod/ofi/ofi_send.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -464,7 +464,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_send(const void *buf, MPI_Aint count, MPI
464464

465465
void *data = NULL;
466466
if (need_pack) {
467-
void *pack_buf = MPL_aligned_alloc(64, data_sz, MPL_MEM_OTHER);
467+
void *pack_buf = MPIDI_OFI_malloc_pack_buffer(sreq, data_sz);
468468
MPIR_ERR_CHKANDJUMP1(pack_buf == NULL, mpi_errno,
469469
MPI_ERR_OTHER, "**nomem", "**nomem %s", "Send Pack buffer alloc");
470470

@@ -475,7 +475,6 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_send(const void *buf, MPI_Aint count, MPI
475475
MPIR_ERR_CHECK(mpi_errno);
476476

477477
data = pack_buf;
478-
MPIDI_OFI_REQUEST(sreq, noncontig.pack.pack_buffer) = pack_buf;
479478
} else {
480479
data = MPIR_get_contig_ptr(buf, dt_true_lb);
481480
MPIDI_OFI_REQUEST(sreq, noncontig.pack.pack_buffer) = NULL;

0 commit comments

Comments
 (0)