prov/udp: detect and use MTU to set max_msg_size and inject_size

For each interface detected by the udp provider, determine the MTU of
the interface, and use that value to set the max_msg_size field of the
fi_ep_attr and fi_tx_attr values of the fi_info element. When the MTU
cannot be determined, the MTU value assumed by previous code
versions (1500) is used.

Signed-off-by: Martin Pokorny <[email protected]>

Author: mpokorny

include/freebsd/osd.h

@@ -76,6 +76,11 @@ static inline size_t ofi_ifaddr_get_speed(struct ifaddrs *ifa)
 	return 0;
 }
 
+static inline int ofi_ifaddr_get_mtu(const struct ifaddrs *ifa)
+{
+	return -1;
+}
+
 static inline ssize_t ofi_process_vm_readv(pid_t pid,
 			const struct iovec *local_iov,
 			unsigned long liovcnt,
@@ -185,5 +190,3 @@ ofi_recvv_socket(SOCKET fd, const struct iovec *iov, size_t cnt, int flags)
 }
 
 #endif /* _FREEBSD_OSD_H_ */
-
-

include/linux/osd.h

@@ -92,6 +92,8 @@ static inline int ofi_hugepage_enabled(void)
 
 size_t ofi_ifaddr_get_speed(struct ifaddrs *ifa);
 
+int ofi_ifaddr_get_mtu(const struct ifaddrs *ifa);
+
 #ifndef __NR_process_vm_readv
 # define __NR_process_vm_readv 310
 #endif

include/ofi_net.h

@@ -655,6 +655,7 @@ struct ofi_addr_list_entry {
 	char			ipstr[INET6_ADDRSTRLEN];
 	union ofi_sock_ip	ipaddr;
 	size_t			speed;
+	int			mtu;
 	char			net_name[OFI_ADDRSTRLEN];
 	char			ifa_name[OFI_ADDRSTRLEN];
 	uint64_t		comm_caps;

include/osx/osd.h

@@ -99,6 +99,11 @@ static inline size_t ofi_ifaddr_get_speed(struct ifaddrs *ifa)
 	return 0;
 }
 
+static inline int ofi_ifaddr_get_mtu(const struct ifaddrs *ifa)
+{
+	return -1;
+}
+
 static inline int ofi_hugepage_enabled(void)
 {
 	return 0;

include/windows/ifaddrs.h

@@ -34,8 +34,8 @@ struct ifaddrs {
 
 	char		   ad_name[16];
 	size_t		   speed;
+	int		   mtu;
 };
 
 int getifaddrs(struct ifaddrs **ifap);
 void freeifaddrs(struct ifaddrs *ifa);
-

include/windows/osd.h

@@ -1006,6 +1006,8 @@ static inline int ofi_is_loopback_addr(struct sockaddr *addr) {
 
 size_t ofi_ifaddr_get_speed(struct ifaddrs *ifa);
 
+int ofi_ifaddr_get_mtu(const struct ifaddrs *ifa);
+
 #define file2unix_time	10000000i64
 #define win2unix_epoch	116444736000000000i64
 #define CLOCK_REALTIME 0

man/fi_udp.7.md

@@ -41,9 +41,8 @@ receiving datagram messages over an unreliable endpoint.
 
 # LIMITATIONS
 
-The UDP provider has hard-coded maximums for supported queue sizes and data
-transfers.  These values are reflected in the related fabric attribute
-structures
+The UDP provider has a hard-coded maximum for supported queue sizes.
+This value is reflected in the related fabric attribute structures.
 
 EPs must be bound to both RX and TX CQs.
 
@@ -53,7 +52,10 @@ No support for counters.
 
 # RUNTIME PARAMETERS
 
-No runtime parameters are currently defined.
+The UDP provider checks for the following environment variables -
+
+*FI_UDP_IFACE*
+: An string value that specifies the name of the interface.
 
 # SEE ALSO
 

prov/udp/src/udpx.h

@@ -63,22 +63,21 @@
 #ifndef _UDPX_H_
 #define _UDPX_H_
 
-
 extern struct fi_provider udpx_prov;
 extern struct util_prov udpx_util_prov;
 extern struct fi_info udpx_info;
 
-
 int udpx_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric,
 		void *context);
 int udpx_domain_open(struct fid_fabric *fabric, struct fi_info *info,
 		struct fid_domain **dom, void *context);
 int udpx_eq_open(struct fid_fabric *fabric, struct fi_eq_attr *attr,
 		struct fid_eq **eq, void *context);
-
+void udpx_util_prov_init(uint32_t version);
 
 #define UDPX_FLAG_MULTI_RECV	1
 #define UDPX_IOV_LIMIT		4
+#define UDPX_MTU		1500
 
 struct udpx_ep_entry {
 	void			*context;
@@ -88,6 +87,8 @@ struct udpx_ep_entry {
 	uint8_t			resv[sizeof(size_t) - 2];
 };
 
+#define UDPX_MAX_MSG_SIZE(mtu) ((mtu) - 28)
+
 OFI_DECLARE_CIRQUE(struct udpx_ep_entry, udpx_rx_cirq);
 
 struct udpx_ep;

prov/udp/src/udpx_attr.c

@@ -31,14 +31,15 @@
  */
 
 #include "udpx.h"
+#include "ofi_osd.h"
 
 #define UDPX_TX_CAPS (OFI_TX_MSG_CAPS | FI_MULTICAST)
 #define UDPX_RX_CAPS (FI_SOURCE | OFI_RX_MSG_CAPS)
 #define UDPX_DOMAIN_CAPS (FI_LOCAL_COMM | FI_REMOTE_COMM)
 
 struct fi_tx_attr udpx_tx_attr = {
 	.caps = UDPX_TX_CAPS,
-	.inject_size = 1472,
+	.inject_size = UDPX_MAX_MSG_SIZE(UDPX_MTU),
 	.size = 1024,
 	.iov_limit = UDPX_IOV_LIMIT
 };
@@ -53,7 +54,7 @@ struct fi_ep_attr udpx_ep_attr = {
 	.type = FI_EP_DGRAM,
 	.protocol = FI_PROTO_UDP,
 	.protocol_version = 0,
-	.max_msg_size = 1472,
+	.max_msg_size = UDPX_MAX_MSG_SIZE(UDPX_MTU),
 	.tx_ctx_cnt = 1,
 	.rx_ctx_cnt = 1
 };
@@ -93,6 +94,62 @@ struct fi_info udpx_info = {
 
 struct util_prov udpx_util_prov = {
 	.prov = &udpx_prov,
-	.info = &udpx_info,
-	.flags = 0,
+	.info = NULL,
+        .flags = 0,
 };
+
+
+static int match_interface(struct slist_entry *entry, const void *infop)
+{
+	struct ofi_addr_list_entry *addr_entry;
+	const struct fi_info* info = infop;
+	addr_entry = container_of(entry, struct ofi_addr_list_entry, entry);
+	return strcmp(addr_entry->net_name, info->fabric_attr->name) == 0 &&
+	       strcmp(addr_entry->ifa_name, info->domain_attr->name) == 0;
+}
+
+static void set_mtu_from_addr_list(struct fi_info* info,
+				   struct slist *addr_list)
+{
+	struct ofi_addr_list_entry *addr_entry;
+	struct slist_entry *entry;
+	int max_msg_size;
+
+	entry = slist_find_first_match(addr_list, match_interface, info);
+	assert(entry);
+	if (entry) {
+		addr_entry = container_of(entry,
+					  struct ofi_addr_list_entry,
+					  entry);
+		max_msg_size = UDPX_MAX_MSG_SIZE(addr_entry->mtu);
+		if (max_msg_size > 0) {
+			info->tx_attr->inject_size = max_msg_size;
+			info->ep_attr->max_msg_size = max_msg_size;
+		}
+	} else {
+		FI_DBG(&udpx_prov, FI_LOG_CORE,
+		       "Failed to match interface (%s, %s) to "
+		       "address for MTU size\n",
+		       info->fabric_attr->name, info->domain_attr->name);
+	}
+}
+
+void udpx_util_prov_init(uint32_t version) {
+
+	struct slist addr_list;
+	struct fi_info* cur;
+	struct fi_info* info;
+
+        if (udpx_util_prov.info == NULL) {
+		udpx_util_prov.info = &udpx_info;
+		info = fi_allocinfo();
+		ofi_ip_getinfo(&udpx_util_prov, version, NULL, NULL, 0, NULL,
+			       &info);
+		slist_init(&addr_list);
+		ofi_get_list_of_addr(&udpx_prov, "iface", &addr_list);
+		for (cur = info; cur; cur = cur->next)
+			set_mtu_from_addr_list(cur, &addr_list);
+		udpx_util_prov.info = info;
+		ofi_free_list_of_addr(&addr_list);
+	}
+}

prov/udp/src/udpx_init.c

@@ -37,20 +37,26 @@
 
 #include <sys/types.h>
 
+static ofi_mutex_t init_lock;
 
 static int udpx_getinfo(uint32_t version, const char *node, const char *service,
 			uint64_t flags, const struct fi_info *hints,
 			struct fi_info **info)
 {
-	return ofi_ip_getinfo(&udpx_util_prov, version, node, service, flags,
-			      hints, info);
+	ofi_mutex_lock(&init_lock);
+	udpx_util_prov_init(version);
+	ofi_mutex_unlock(&init_lock);
+	return util_getinfo(&udpx_util_prov, version, node, service, flags,
+			    hints, info);
 }
 
 static void udpx_fini(void)
 {
-	/* yawn */
+	if (udpx_util_prov.info != NULL)
+		fi_freeinfo(udpx_util_prov.info);
 }
 
+
 struct fi_provider udpx_prov = {
 	.name = "udp",
 	.version = OFI_VERSION_DEF_PROV,
@@ -65,5 +71,6 @@ UDP_INI
 	fi_param_define(&udpx_prov, "iface", FI_PARAM_STRING,
 			"Specify interface name");
 
+	ofi_mutex_init(&init_lock);
 	return &udpx_prov;
 }