Skip to content

Select device lanes #11015

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+67 −17
Open

Select device lanes #11015

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
41ffa64
UCP/DEVICE: Select lane.
ofirfarjun7 Nov 19, 2025
7939f57
UCP/DEVICE: Fix.
ofirfarjun7 Nov 19, 2025
00de4e5
UCP/DEVICE: CR comments.
ofirfarjun7 Nov 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 67 additions & 17 deletions src/ucp/core/ucp_device.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,18 +280,45 @@ ucp_device_mem_list_params_check(ucp_context_h context,
return UCS_OK;
}

typedef struct {
double bandwidth;
ucp_lane_index_t lane;
const char *dev_name;
} ucp_device_lane_info_t;

static int ucp_device_lane_compare(const void *a, const void *b)
{
const ucp_device_lane_info_t *lane_a = a;
const ucp_device_lane_info_t *lane_b = b;
int cmp;

cmp = -ucp_score_cmp(lane_a->bandwidth, lane_b->bandwidth);
if (cmp != 0) {
return cmp;
}

/* If bandwidth is equal, sort by device name */
return strcmp(lane_a->dev_name, lane_b->dev_name);
}

static void ucp_device_mem_list_lane_lookup(
ucp_ep_h ep, ucp_ep_config_t *ep_config, ucs_sys_device_t local_sys_dev,
ucp_md_map_t local_md_map, ucs_sys_device_t remote_sys_dev,
ucp_md_map_t remote_md_map,
ucp_lane_index_t lanes[UCP_DEVICE_MEM_LIST_MAX_EPS])
{
double best_bw[UCP_DEVICE_MEM_LIST_MAX_EPS] = {-1., -1.};
unsigned long node_local_id = ep->worker->context->config.node_local_id;
Copy link
Contributor

@iyastreb iyastreb Nov 21, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

align

Copy link
Contributor Author

@ofirfarjun7 ofirfarjun7 Nov 25, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I run format script and it kept it as is.

ucp_lane_index_t num_valid_rma_lanes = 0;
ucp_lane_index_t best_non_rma_lane = UCP_NULL_LANE;
double best_non_rma_lane_bw = -1.0;
ucp_device_lane_info_t valid_rma_lanes[UCP_MAX_LANES];
ucp_lane_index_t lane;
ucp_device_lane_info_t *selected_rma_lane;
double bandwidth;
ucp_ep_config_key_lane_t *lane_key;
ucs_sys_device_t src_sys_dev;
ucp_md_index_t src_md_index;
int is_rma;

lanes[0] = UCP_NULL_LANE;
lanes[1] = UCP_NULL_LANE;
Expand Down Expand Up @@ -331,26 +358,49 @@ static void ucp_device_mem_list_lane_lookup(

bandwidth = ucp_worker_iface_bandwidth(ep->worker,
ucp_ep_get_rsc_index(ep, lane));

is_rma = !!(ucp_ep_md_attr(ep, lane)->flags & UCT_MD_FLAG_NEED_MEMH);
Copy link
Contributor

@yosefe yosefe Nov 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. not sure we need the separation between RMA and non-RMA lanes
  2. we could just save the indexes of the valid lanes and then sort, and not save all lanes properties in the array. Use ucs_qsort_r to pass context to get the rest of the info needed for comparison.
  3. can use ucs_array

ucs_trace("checking lane[%u] src_md_index=%u dst_md_index=%u "
"src_sys_dev=%u dst_sys_dev=%u bandwidth=%lfMB/s",
"src_sys_dev=%u dst_sys_dev=%u bandwidth=%lfMB/s is_rma=%d",
lane, src_md_index, lane_key->dst_md_index, src_sys_dev,
lane_key->dst_sys_dev, bandwidth / UCS_MBYTE);

UCS_STATIC_ASSERT(UCP_DEVICE_MEM_LIST_MAX_EPS == 2);
if (bandwidth > best_bw[0]) {
best_bw[1] = best_bw[0];
lanes[1] = lanes[0];
best_bw[0] = bandwidth;
lanes[0] = lane;
} else if (bandwidth > best_bw[1]) {
best_bw[1] = bandwidth;
lanes[1] = lane;
} else {
continue;
lane_key->dst_sys_dev, bandwidth / UCS_MBYTE, is_rma);

if (is_rma) {
valid_rma_lanes[num_valid_rma_lanes].bandwidth = bandwidth;
valid_rma_lanes[num_valid_rma_lanes].lane = lane;
valid_rma_lanes[num_valid_rma_lanes].dev_name =
ucp_ep_get_tl_rsc(ep, lane)->dev_name;
num_valid_rma_lanes++;
} else if ((best_non_rma_lane == UCP_NULL_LANE) ||
(bandwidth > best_non_rma_lane_bw)) {
best_non_rma_lane = lane;
best_non_rma_lane_bw = bandwidth;
}
}

if (best_non_rma_lane != UCP_NULL_LANE) {
lanes[0] = best_non_rma_lane;
ucs_trace("selected non-RMA lane: (%lfMB/s)",
best_non_rma_lane_bw / UCS_MBYTE);
}

if (num_valid_rma_lanes == 0) {
return;
}

ucs_trace("best lanes: lane[%u]=%lfMB/s lane[%u]=%lfMB/s", lanes[0],
best_bw[0] / UCS_MBYTE, lanes[1], best_bw[1] / UCS_MBYTE);
/* Sort RMA lanes */
qsort(valid_rma_lanes, num_valid_rma_lanes, sizeof(ucp_device_lane_info_t),
ucp_device_lane_compare);
selected_rma_lane = &valid_rma_lanes[node_local_id % num_valid_rma_lanes];
ucs_trace("selected RMA lane: (idx=%u, %lfMB/s) "
"node_local_id=%lu dev_name=%s",
selected_rma_lane->lane, selected_rma_lane->bandwidth / UCS_MBYTE,
node_local_id, selected_rma_lane->dev_name);
if (selected_rma_lane->bandwidth > best_non_rma_lane_bw) {
lanes[1] = lanes[0];
lanes[0] = selected_rma_lane->lane;
} else {
lanes[1] = selected_rma_lane->lane;
}
}

Expand Down
Loading