Skip to content

fabtests/benchmarks: Separate warmup and measurement phases in RMA ba… #11593

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
+31 −7
Draft

fabtests/benchmarks: Separate warmup and measurement phases in RMA ba… #11593

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 24 additions & 4 deletions fabtests/benchmarks/benchmark_shared.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -658,8 +658,9 @@ static int bw_rma_comp(enum ft_rma_opcodes rma_op, int num_completions)
return ret;
}

int bandwidth_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
int bandwidth_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote, bool is_warmup)
{
const int num_iterations = is_warmup ? opts.warmup_iterations : opts.iterations;
int ret, i, j;
uint64_t flags = 0;
size_t offset, inject_size = fi->tx_attr->inject_size;
Expand Down Expand Up @@ -688,9 +689,8 @@ int bandwidth_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)

offset_rma_start = FT_RMA_SYNC_MSG_BYTES +
MAX(ft_tx_prefix_size(), ft_rx_prefix_size());
for (i = j = 0; i < opts.iterations + opts.warmup_iterations; i++) {
if (i == opts.warmup_iterations)
ft_start();
ft_start();
for (i = j = 0; i < num_iterations; i++) {
if (j == 0) {
offset = offset_rma_start;
if (ft_check_opts(FT_OPT_VERIFY_DATA) && opts.transfer_size > 0) {
Expand Down Expand Up @@ -726,6 +726,16 @@ int bandwidth_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
break;
case FT_RMA_WRITEDATA:
if (!opts.dst_addr) {
if (is_warmup) {
ret = ft_post_tx(
ep,
remote_fi_addr,
FT_RMA_SYNC_MSG_BYTES,
NO_CQ_DATA,
&tx_ctx);
if (ret)
return ret;
}
if (fi->rx_attr->mode & FI_RX_CQ_DATA)
ret = ft_post_rx(ep, 0, &rx_ctx_arr[j].context);
else
Expand All @@ -736,6 +746,14 @@ int bandwidth_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
rx_seq++;

} else {
if (is_warmup) {
ret = ft_post_rx(
ep,
FT_RMA_SYNC_MSG_BYTES,
&rx_ctx);
if (ret)
return ret;
}
if (opts.transfer_size <= inject_size) {
ret = ft_post_rma_inject(FT_RMA_WRITEDATA,
tx_buf + offset,
Expand Down Expand Up @@ -780,6 +798,8 @@ int bandwidth_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
return ret;
ft_stop();

if (is_warmup)
return 0;
if (opts.machr)
show_perf_mr(opts.transfer_size, opts.iterations, &start, &end, 1,
opts.argc, opts.argv);
Expand Down
2 changes: 1 addition & 1 deletion fabtests/benchmarks/benchmark_shared.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ int pingpong(void);
int run_pingpong(void);
int bandwidth(void);
int pingpong_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote);
int bandwidth_rma(enum ft_rma_opcodes op, struct fi_rma_iov *remote);
int bandwidth_rma(enum ft_rma_opcodes op, struct fi_rma_iov *remote, bool is_warmup);
int rma_tx_completion(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote);

#ifdef __cplusplus
Expand Down
8 changes: 6 additions & 2 deletions fabtests/benchmarks/rma_bw.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,19 +58,23 @@ static int run(void)
if (ret)
return ret;

ret = bandwidth_rma(opts.rma_op, &remote, true);
if (ret)
goto out;

if (!(opts.options & FT_OPT_SIZE)) {
for (i = 0; i < TEST_CNT; i++) {
if (!ft_use_size(i, opts.sizes_enabled))
continue;
opts.transfer_size = test_size[i].size;
init_test(&opts, test_name, sizeof(test_name));
ret = bandwidth_rma(opts.rma_op, &remote);
ret = bandwidth_rma(opts.rma_op, &remote, false);
if (ret)
goto out;
}
} else {
init_test(&opts, test_name, sizeof(test_name));
ret = bandwidth_rma(opts.rma_op, &remote);
ret = bandwidth_rma(opts.rma_op, &remote, false);
if (ret)
goto out;
}
Expand Down