Skip to content

Commit ebe3a5f

Browse files
alekswnalekswn
committed
fabtests: implement pausable timer
Replace global timespec variables with a flexible ft_timer union that supports pause/resume functionality. This allows for more accurate timing measurements in tests by excluding syncronization time. Key changes: - Add ft_timer union with pause/resume capabilities - Replace timespec usage across all test files with new timer API - Update timing functions in benchmark and test utilities - `bandwidth_rma -o writedata` exclude reverse sync time from timing - Fix a bug in `uni_bandwidth` due to dafaulting to a global var `targs` - Fixed a bug with timing in `efa_gda` fabtest This improves timing precision in fabtests by allowing tests to exclude non-measurement periods from their timing calculations. Signed-off-by: Alexey Novikov <[email protected]>
1 parent 5b39be8 commit ebe3a5f

File tree

16 files changed

+218
-188
lines changed

16 files changed

+218
-188
lines changed

‎fabtests/benchmarks/benchmark_shared.c‎

Lines changed: 56 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -94,15 +94,14 @@ void ft_benchmark_usage(void)
9494
}
9595

9696
/* Pingpong latency test with pre-posted receive buffers. */
97-
static int pingpong_pre_posted_rx(size_t inject_size)
97+
static int pingpong_pre_posted_rx(size_t inject_size, union ft_timer *timer)
9898
{
9999
int ret, i;
100100

101101
if (opts.dst_addr) {
102102
for (i = 0; i < opts.iterations + opts.warmup_iterations; i++) {
103103
if (i == opts.warmup_iterations)
104-
ft_start();
105-
104+
ft_start(timer);
106105
if (opts.transfer_size <= inject_size)
107106
ret = ft_inject(ep, remote_fi_addr,
108107
opts.transfer_size);
@@ -119,7 +118,7 @@ static int pingpong_pre_posted_rx(size_t inject_size)
119118
} else {
120119
for (i = 0; i < opts.iterations + opts.warmup_iterations; i++) {
121120
if (i == opts.warmup_iterations)
122-
ft_start();
121+
ft_start(timer);
123122

124123
ret = ft_rx(ep, opts.transfer_size);
125124
if (ret)
@@ -135,20 +134,20 @@ static int pingpong_pre_posted_rx(size_t inject_size)
135134
return ret;
136135
}
137136
}
138-
ft_stop();
137+
ft_stop(timer);
139138

140139
return FI_SUCCESS;
141140
}
142141

143142
/* Pingpong latency test without pre-posted receive buffers. */
144-
static int pingpong_no_pre_posted_rx(size_t inject_size)
143+
static int pingpong_no_pre_posted_rx(size_t inject_size, union ft_timer *timer)
145144
{
146145
int ret, i;
147146

148147
if (opts.dst_addr) {
149148
for (i = 0; i < opts.iterations + opts.warmup_iterations; i++) {
150149
if (i == opts.warmup_iterations)
151-
ft_start();
150+
ft_start(timer);
152151

153152
if (opts.transfer_size <= inject_size)
154153
ret = ft_inject(ep, remote_fi_addr,
@@ -170,7 +169,7 @@ static int pingpong_no_pre_posted_rx(size_t inject_size)
170169
} else {
171170
for (i = 0; i < opts.iterations + opts.warmup_iterations; i++) {
172171
if (i == opts.warmup_iterations)
173-
ft_start();
172+
ft_start(timer);
174173

175174
ret = ft_post_rx(ep, opts.transfer_size, &rx_ctx);
176175
if (ret)
@@ -197,14 +196,15 @@ static int pingpong_no_pre_posted_rx(size_t inject_size)
197196
return ret;
198197
}
199198
}
200-
ft_stop();
199+
ft_stop(timer);
201200

202201
return FI_SUCCESS;
203202
}
204203

205204
int pingpong(void)
206205
{
207206
int ret;
207+
union ft_timer timer = {};
208208
size_t inject_size = fi->tx_attr->inject_size;
209209

210210
ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_INJECT_MSG_SIZE,
@@ -236,24 +236,24 @@ int pingpong(void)
236236
return ret;
237237
}
238238

239-
ret = pingpong_no_pre_posted_rx(inject_size);
239+
ret = pingpong_no_pre_posted_rx(inject_size, &timer);
240240
if (ret)
241241
return ret;
242242
} else {
243243
ret = ft_sync();
244244
if (ret)
245245
return ret;
246246

247-
ret = pingpong_pre_posted_rx(inject_size);
247+
ret = pingpong_pre_posted_rx(inject_size, &timer);
248248
if (ret)
249249
return ret;
250250
}
251251

252252
if (opts.machr)
253-
show_perf_mr(opts.transfer_size, opts.iterations, &start, &end, 2,
253+
show_perf_mr(opts.transfer_size, opts.iterations, timer, 2,
254254
opts.argc, opts.argv);
255255
else
256-
show_perf(NULL, opts.transfer_size, opts.iterations, &start, &end, 2);
256+
show_perf(NULL, opts.transfer_size, opts.iterations, timer, 2);
257257

258258
return 0;
259259
}
@@ -296,6 +296,7 @@ int run_pingpong(void)
296296
int pingpong_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
297297
{
298298
int ret, i;
299+
union ft_timer timer = {};
299300
size_t inject_size = fi->tx_attr->inject_size;
300301

301302
ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_INJECT_RMA_SIZE,
@@ -335,7 +336,7 @@ int pingpong_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
335336
for (i = 0; i < opts.iterations + opts.warmup_iterations; i++) {
336337

337338
if (i == opts.warmup_iterations)
338-
ft_start();
339+
ft_start(&timer);
339340

340341
if (rma_op == FT_RMA_WRITE)
341342
*(tx_buf + opts.transfer_size - 1) = (char)i;
@@ -357,7 +358,7 @@ int pingpong_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
357358
} else {
358359
for (i = 0; i < opts.iterations + opts.warmup_iterations; i++) {
359360
if (i == opts.warmup_iterations)
360-
ft_start();
361+
ft_start(&timer);
361362

362363
ret = ft_rx_rma(i, rma_op, ep, opts.transfer_size);
363364
if (ret)
@@ -377,20 +378,21 @@ int pingpong_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
377378
return ret;
378379
}
379380
}
380-
ft_stop();
381+
ft_stop(&timer);
381382

382383
if (opts.machr)
383-
show_perf_mr(opts.transfer_size, opts.iterations, &start, &end, 2,
384+
show_perf_mr(opts.transfer_size, opts.iterations, timer, 2,
384385
opts.argc, opts.argv);
385386
else
386-
show_perf(NULL, opts.transfer_size, opts.iterations, &start, &end, 2);
387+
show_perf(NULL, opts.transfer_size, opts.iterations, timer, 2);
387388

388389
return 0;
389390
}
390391

391392
int rma_tx_completion(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
392393
{
393394
int ret, i;
395+
union ft_timer timer = {};
394396
size_t inject_size = fi->tx_attr->inject_size;
395397

396398
ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_INJECT_RMA_SIZE,
@@ -418,7 +420,7 @@ int rma_tx_completion(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
418420
if (opts.dst_addr) {
419421
for (i = 0; i < opts.iterations + opts.warmup_iterations; i++) {
420422
if (i == opts.warmup_iterations)
421-
ft_start();
423+
ft_start(&timer);
422424

423425
if (opts.transfer_size <= inject_size)
424426
ret = ft_inject_rma(rma_op, remote, ep,
@@ -431,12 +433,12 @@ int rma_tx_completion(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
431433
return ret;
432434
}
433435

434-
ft_stop();
436+
ft_stop(&timer);
435437
if (opts.machr)
436-
show_perf_mr(opts.transfer_size, opts.iterations, &start, &end, 1,
438+
show_perf_mr(opts.transfer_size, opts.iterations, timer, 1,
437439
opts.argc, opts.argv);
438440
else
439-
show_perf(NULL, opts.transfer_size, opts.iterations, &start, &end, 1);
441+
show_perf(NULL, opts.transfer_size, opts.iterations, timer, 1);
440442

441443
/* Inform RMA target that the test has ended */
442444
ret = ft_sync();
@@ -461,14 +463,19 @@ int rma_tx_completion(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
461463
return 0;
462464
}
463465

464-
static int bw_tx_comp()
466+
static int bw_tx_comp(union ft_timer *timer)
465467
{
466468
int ret;
467469

468470
ret = ft_get_tx_comp(tx_seq);
469471
if (ret)
470472
return ret;
471-
return ft_rx(ep, FT_RMA_SYNC_MSG_BYTES);
473+
if (timer)
474+
ft_timer_pause(timer);
475+
ret = ft_rx(ep, FT_RMA_SYNC_MSG_BYTES);
476+
if (timer)
477+
ft_timer_resume(timer);
478+
return ret;
472479
}
473480

474481
static int bw_rx_comp(int window)
@@ -493,16 +500,20 @@ static int bw_rx_comp(int window)
493500
return ft_tx(ep, remote_fi_addr, FT_RMA_SYNC_MSG_BYTES, &tx_ctx);
494501
}
495502

496-
static int rma_bw_rx_comp()
503+
static int rma_bw_rx_comp(union ft_timer *timer)
497504
{
498505
int ret;
499506

500507
/* rx_seq is always one ahead */
501508
ret = ft_get_rx_comp(rx_seq - 1);
502509
if (ret)
503510
return ret;
504-
505-
return ft_tx(ep, remote_fi_addr, FT_RMA_SYNC_MSG_BYTES, &tx_ctx);
511+
if (timer)
512+
ft_timer_pause(timer);
513+
ret = ft_tx(ep, remote_fi_addr, FT_RMA_SYNC_MSG_BYTES, &tx_ctx);
514+
if (timer)
515+
ft_timer_resume(timer);
516+
return ret;
506517
}
507518

508519
static uint64_t set_fi_more_flag(int i, int j, uint64_t flags)
@@ -520,6 +531,7 @@ int bandwidth(void)
520531
{
521532
int ret, i, j;
522533
uint64_t flags = 0;
534+
union ft_timer timer = {};
523535
size_t inject_size = fi->tx_attr->inject_size;
524536

525537
ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_INJECT_MSG_SIZE,
@@ -549,7 +561,7 @@ int bandwidth(void)
549561
if (opts.dst_addr) {
550562
for (i = j = 0; i < opts.iterations + opts.warmup_iterations; i++) {
551563
if (i == opts.warmup_iterations)
552-
ft_start();
564+
ft_start(&timer);
553565

554566
if (ft_check_opts(FT_OPT_VERIFY_DATA)) {
555567
ret = ft_fill_buf(tx_ctx_arr[j].buf,
@@ -578,19 +590,19 @@ int bandwidth(void)
578590
return ret;
579591

580592
if (++j == opts.window_size) {
581-
ret = bw_tx_comp();
593+
ret = bw_tx_comp(NULL);
582594
if (ret)
583595
return ret;
584596
j = 0;
585597
}
586598
}
587-
ret = bw_tx_comp();
599+
ret = bw_tx_comp(NULL);
588600
if (ret)
589601
return ret;
590602
} else {
591603
for (i = j = 0; i < opts.iterations + opts.warmup_iterations; i++) {
592604
if (i == opts.warmup_iterations)
593-
ft_start();
605+
ft_start(&timer);
594606

595607
if (opts.use_fi_more) {
596608
flags = set_fi_more_flag(i, j, flags);
@@ -620,27 +632,27 @@ int bandwidth(void)
620632
if (ret)
621633
return ret;
622634
}
623-
ft_stop();
635+
ft_stop(&timer);
624636

625637
if (opts.machr)
626-
show_perf_mr(opts.transfer_size, opts.iterations, &start, &end, 1,
638+
show_perf_mr(opts.transfer_size, opts.iterations, timer, 1,
627639
opts.argc, opts.argv);
628640
else
629-
show_perf(NULL, opts.transfer_size, opts.iterations, &start, &end, 1);
641+
show_perf(NULL, opts.transfer_size, opts.iterations, timer, 1);
630642

631643
return 0;
632644
}
633645

634-
static int bw_rma_comp(enum ft_rma_opcodes rma_op, int num_completions)
646+
static int bw_rma_comp(enum ft_rma_opcodes rma_op, int num_completions, union ft_timer *timer)
635647
{
636648
int ret;
637649

638650
if (rma_op == FT_RMA_WRITEDATA) {
639651
/* for writedata, only the client sends,
640652
* and only the server verifies. */
641653
if (opts.dst_addr)
642-
return bw_tx_comp();
643-
ret = rma_bw_rx_comp();
654+
return bw_tx_comp(timer);
655+
ret = rma_bw_rx_comp(timer);
644656
} else {
645657
ret = ft_get_tx_comp(tx_seq);
646658
}
@@ -662,6 +674,7 @@ int bandwidth_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
662674
{
663675
int ret, i, j;
664676
uint64_t flags = 0;
677+
union ft_timer timer = {};
665678
size_t offset, inject_size = fi->tx_attr->inject_size;
666679

667680
ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_INJECT_RMA_SIZE,
@@ -690,7 +703,7 @@ int bandwidth_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
690703
MAX(ft_tx_prefix_size(), ft_rx_prefix_size());
691704
for (i = j = 0; i < opts.iterations + opts.warmup_iterations; i++) {
692705
if (i == opts.warmup_iterations)
693-
ft_start();
706+
ft_start(&timer);
694707
if (j == 0) {
695708
offset = offset_rma_start;
696709
if (ft_check_opts(FT_OPT_VERIFY_DATA) && opts.transfer_size > 0) {
@@ -768,22 +781,22 @@ int bandwidth_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
768781
return ret;
769782

770783
if (++j == opts.window_size) {
771-
ret = bw_rma_comp(rma_op, j);
784+
ret = bw_rma_comp(rma_op, j, &timer);
772785
if (ret)
773786
return ret;
774787
j = 0;
775788
}
776789
offset += opts.transfer_size;
777790
}
778-
ret = bw_rma_comp(rma_op, j);
791+
ret = bw_rma_comp(rma_op, j, &timer);
779792
if (ret)
780793
return ret;
781-
ft_stop();
794+
ft_stop(&timer);
782795

783796
if (opts.machr)
784-
show_perf_mr(opts.transfer_size, opts.iterations, &start, &end, 1,
797+
show_perf_mr(opts.transfer_size, opts.iterations, timer, 1,
785798
opts.argc, opts.argv);
786799
else
787-
show_perf(NULL, opts.transfer_size, opts.iterations, &start, &end, 1);
800+
show_perf(NULL, opts.transfer_size, opts.iterations, timer, 1);
788801
return 0;
789802
}

0 commit comments

Comments
 (0)