prov/efa: Use endpoint->peer hashmap at the AV level

prov/efa/src/efa_av.c

@@ -341,12 +341,11 @@ static int efa_conn_implicit_to_explicit(struct efa_av *av,
 {
 	int err;
 	struct efa_ah *ah;
-	fi_addr_t shm_fi_addr;
-	struct efa_conn *conn;
+	struct efa_conn *implicit_conn, *explicit_conn;
 	struct efa_rdm_ep *ep;
-	struct efa_rdm_peer *peer;
 	struct dlist_entry *entry;
-	struct util_av_entry *util_av_entry;
+	struct util_av_entry *implicit_util_av_entry, *explicit_util_av_entry;
+	struct efa_conn_ep_peer_map_entry *map_entry, *tmp;
 	struct efa_av_entry *implicit_av_entry, *explicit_av_entry;
 	struct fid_peer_srx *peer_srx;
 
@@ -358,97 +357,98 @@ static int efa_conn_implicit_to_explicit(struct efa_av *av,
 	assert(ofi_genlock_held(&av->util_av.lock));
 	assert(ofi_genlock_held(&av->util_av_implicit.lock));
 
-	util_av_entry =
+	/* Get implicit util AV entry and conn */
+	implicit_util_av_entry =
 	ofi_bufpool_get_ibuf(av->util_av_implicit.av_entry_pool, implicit_fi_addr);
 
-	implicit_av_entry = (struct efa_av_entry *) util_av_entry->data;
+	implicit_av_entry = (struct efa_av_entry *) implicit_util_av_entry->data;
 
 	assert(implicit_av_entry);
 	assert(efa_is_same_addr(
 		raw_addr, (struct efa_ep_addr *) implicit_av_entry->ep_addr));
 
-	conn = &implicit_av_entry->conn;
-	assert(conn->fi_addr == FI_ADDR_NOTAVAIL &&
-	       conn->implicit_fi_addr == implicit_fi_addr);
+	implicit_conn = &implicit_av_entry->conn;
+	assert(implicit_conn->fi_addr == FI_ADDR_NOTAVAIL &&
+	       implicit_conn->implicit_fi_addr == implicit_fi_addr);
 
-	ah = conn->ah;
-	assert(ah);
-	shm_fi_addr = implicit_av_entry->conn.shm_fi_addr;
-
-	efa_av_reverse_av_remove(&av->cur_reverse_av_implicit,
-				 &av->prv_reverse_av_implicit, conn);
+	ah = implicit_conn->ah;
 
-	err = ofi_av_remove_addr(&av->util_av_implicit, implicit_fi_addr);
+	/* Create explicit util AV entry and conn */
+	err = ofi_av_insert_addr(&av->util_av, raw_addr, fi_addr);
 	if (err) {
 		EFA_WARN(FI_LOG_AV,
-			 "ofi_av_remove_addr from implicit AV failed! Error "
+			 "ofi_av_insert_addr into explicit AV failed! Error "
 			 "message: %s\n",
 			 fi_strerror(err));
 		return err;
 	}
 
-	dlist_remove(&implicit_av_entry->conn.implicit_av_lru_entry);
+	explicit_util_av_entry =
+		ofi_bufpool_get_ibuf(av->util_av.av_entry_pool, *fi_addr);
+	explicit_av_entry = (struct efa_av_entry *) explicit_util_av_entry->data;
+	assert(efa_is_same_addr(
+		raw_addr, (struct efa_ep_addr *) explicit_av_entry->ep_addr));
 
-	assert(!dlist_empty(&conn->ah->implicit_conn_list));
-	dlist_remove(&conn->ah_implicit_conn_list_entry);
-	efa_ah_implicit_av_lru_ah_move(av->domain, conn->ah);
+	/* Copy information from implicit conn to explicit conn */
+	explicit_conn = &explicit_av_entry->conn;
+	memset(explicit_conn, 0, sizeof(*explicit_conn));
+	explicit_conn->ep_addr = (struct efa_ep_addr *) explicit_av_entry->ep_addr;
+	assert(av->type == FI_AV_TABLE);
+	explicit_conn->ah = implicit_conn->ah;
+	explicit_conn->fi_addr = *fi_addr;
+	explicit_conn->shm_fi_addr = implicit_conn->shm_fi_addr;
+	explicit_conn->implicit_fi_addr = FI_ADDR_NOTAVAIL;
+	HASH_ITER(hh, implicit_conn->ep_peer_map, map_entry, tmp) {
+		HASH_DELETE(hh, implicit_conn->ep_peer_map, map_entry);
+		HASH_ADD_PTR(explicit_conn->ep_peer_map, ep_ptr, map_entry);
+		map_entry->peer.conn = explicit_conn;
+	}
+	assert(HASH_CNT(hh, implicit_conn->ep_peer_map) == 0);
 
-	av->used_implicit--;
-	conn->ah->implicit_refcnt--;
+	/* Handle reverse AV and AV ref counts */
+	efa_av_reverse_av_remove(&av->cur_reverse_av_implicit,
+				 &av->prv_reverse_av_implicit, implicit_conn);
 
-	err = ofi_av_insert_addr(&av->util_av, raw_addr, fi_addr);
+	dlist_remove(&implicit_av_entry->conn.implicit_av_lru_entry);
+
+	err = ofi_av_remove_addr(&av->util_av_implicit, implicit_fi_addr);
 	if (err) {
 		EFA_WARN(FI_LOG_AV,
-			 "ofi_av_insert_addr into explicit AV failed! Error "
+			 "ofi_av_remove_addr from implicit AV failed! Error "
 			 "message: %s\n",
 			 fi_strerror(err));
 		return err;
 	}
 
-	util_av_entry =
-		ofi_bufpool_get_ibuf(av->util_av.av_entry_pool, *fi_addr);
-	explicit_av_entry = (struct efa_av_entry *) util_av_entry->data;
-	assert(efa_is_same_addr(
-		raw_addr, (struct efa_ep_addr *) explicit_av_entry->ep_addr));
-
-	conn = &explicit_av_entry->conn;
-	memset(conn, 0, sizeof(*conn));
-	conn->ep_addr = (struct efa_ep_addr *) explicit_av_entry->ep_addr;
-	assert(av->type == FI_AV_TABLE);
-	conn->ah = ah;
-	conn->fi_addr = *fi_addr;
-	conn->shm_fi_addr = shm_fi_addr;
-	conn->implicit_fi_addr = FI_ADDR_NOTAVAIL;
+	av->used_implicit--;
 
 	err = efa_av_reverse_av_add(av, &av->cur_reverse_av, &av->prv_reverse_av,
-				    conn);
+				    explicit_conn);
 	if (err)
 		return err;
 
 	av->used_explicit++;
-	conn->ah->explicit_refcnt++;
+
+	/* Handle AH LRU list and refcnt */
+	assert(!dlist_empty(&ah->implicit_conn_list));
+	dlist_remove(&implicit_conn->ah_implicit_conn_list_entry);
+	efa_ah_implicit_av_lru_ah_move(av->domain, ah);
+	ah->implicit_refcnt--;
+	ah->explicit_refcnt++;
 
 	EFA_INFO(FI_LOG_AV,
 		 "Peer with implicit fi_addr %" PRIu64
 		 " moved to explicit AV. Explicit fi_addr: %" PRIu64 "\n",
 		 implicit_fi_addr, *fi_addr);
 
-	/* Move peer from implicit peer map to explicit peer map for all
-	 * endpoints. Also call foreach_unspec_addr to move unexpected messages
+	/* Call foreach_unspec_addr to move unexpected messages
 	 * from the unspecified queue to the specified queues
 	 *
 	 * util_ep is bound to the explicit util_av, so the explicit util_av's
 	 * ep_list contains all of the endpoints bound to this AV */
 	ofi_genlock_lock(&av->util_av.ep_list_lock);
 	dlist_foreach(&av->util_av.ep_list, entry) {
 		ep = container_of(entry, struct efa_rdm_ep, base_ep.util_ep.av_entry);
-		peer = efa_rdm_ep_peer_map_lookup(&ep->fi_addr_to_peer_map_implicit, implicit_fi_addr);
-		if (peer) {
-			peer->conn = conn;
-			EFA_INFO(FI_LOG_AV, "Moving peer from implicit to explicit peer map for endpoint %p\n", ep);
-			efa_rdm_ep_peer_map_implicit_to_explicit(ep, peer, implicit_fi_addr, *fi_addr);
-		}
-
 		peer_srx = util_get_peer_srx(ep->peer_srx_ep);
 		peer_srx->owner_ops->foreach_unspec_addr(peer_srx, &efa_av_get_addr_from_peer_rx_entry);
 	}
@@ -800,8 +800,9 @@ static int efa_av_close(struct fid *fid)
 		if (av->shm_rdm_av) {
 			err = fi_close(&av->shm_rdm_av->fid);
 			if (OFI_UNLIKELY(err)) {
-				EFA_WARN(FI_LOG_AV, "Failed to close shm av: %s\n",
-					fi_strerror(err));
+				EFA_WARN(FI_LOG_AV,
+					 "Failed to close shm av: %s\n",
+					 fi_strerror(err));
 			}
 		}
 	}

prov/efa/src/efa_conn.c

@@ -174,11 +174,7 @@ int efa_conn_rdm_insert_shm_av(struct efa_av *av, struct efa_conn *conn)
 void efa_conn_rdm_deinit(struct efa_av *av, struct efa_conn *conn)
 {
 	int err;
-	struct efa_rdm_peer *peer;
-	struct efa_rdm_ep *ep;
-	struct dlist_entry *entry, *tmp;
-	fi_addr_t fi_addr;
-	struct efa_rdm_ep_peer_map_entry **peer_map;
+	struct efa_conn_ep_peer_map_entry *peer_map_entry, *tmp;
 
 	assert(av->domain->info_type == EFA_INFO_RDM);
 
@@ -200,24 +196,13 @@ void efa_conn_rdm_deinit(struct efa_av *av, struct efa_conn *conn)
 	}
 
 	assert(ofi_genlock_held(&av->domain->srx_lock));
-	dlist_foreach_safe (&av->util_av.ep_list, entry, tmp) {
-		ep = container_of(entry, struct efa_rdm_ep,
-				  base_ep.util_ep.av_entry);
-
-		if (conn->fi_addr != FI_ADDR_NOTAVAIL) {
-			peer_map = &ep->fi_addr_to_peer_map;
-			fi_addr = conn->fi_addr;
-		} else {
-			peer_map = &ep->fi_addr_to_peer_map_implicit;
-			fi_addr = conn->implicit_fi_addr;
-		}
-
-		peer = efa_rdm_ep_peer_map_lookup(peer_map, fi_addr);
-		if (peer) {
-			efa_rdm_peer_destruct(peer, ep);
-			efa_rdm_ep_peer_map_remove(peer_map, fi_addr);
-		}
+	HASH_ITER(hh, conn->ep_peer_map, peer_map_entry, tmp) {
+		dlist_remove(&peer_map_entry->peer.ep_peer_list_entry);
+		efa_rdm_peer_destruct(&peer_map_entry->peer, peer_map_entry->ep_ptr);
+		HASH_DEL(conn->ep_peer_map, peer_map_entry);
+		ofi_buf_free(peer_map_entry);
 	}
+	assert(HASH_CNT(hh, conn->ep_peer_map) == 0);
 }
 
 /**
@@ -466,3 +451,30 @@ void efa_conn_release_ah_unsafe(struct efa_av *av, struct efa_conn *conn,
 
 	release_from_implicit_av ? av->used_implicit-- : av->used_explicit--;
 }
+
+int efa_conn_ep_peer_map_insert(struct efa_conn *conn, struct efa_conn_ep_peer_map_entry *map_entry)
+{
+	HASH_ADD_PTR(conn->ep_peer_map, ep_ptr, map_entry);
+
+	return FI_SUCCESS;
+}
+
+struct efa_rdm_peer *efa_conn_ep_peer_map_lookup(struct efa_conn *conn,
+						 struct efa_rdm_ep *ep)
+{
+	struct efa_conn_ep_peer_map_entry *map_entry;
+
+	HASH_FIND_PTR(conn->ep_peer_map, &ep, map_entry);
+
+	return map_entry ? &map_entry->peer : NULL;
+}
+
+void efa_conn_ep_peer_map_remove(struct efa_conn *conn, struct efa_rdm_ep *ep)
+{
+	struct efa_conn_ep_peer_map_entry *map_entry;
+
+	HASH_FIND_PTR(conn->ep_peer_map, &ep, map_entry);
+	assert(map_entry);
+	HASH_DELETE(hh, conn->ep_peer_map, map_entry);
+	ofi_buf_free(map_entry);
+}

prov/efa/src/efa_conn.h

@@ -5,6 +5,7 @@
 #define EFA_CONN_H
 
 #include "ofi_util.h"
+#include "efa_rdm_peer.h"
 
 struct efa_conn {
 	struct efa_ah *ah;
@@ -15,8 +16,23 @@ struct efa_conn {
 	fi_addr_t		shm_fi_addr;
 	struct dlist_entry	implicit_av_lru_entry;
 	struct dlist_entry ah_implicit_conn_list_entry;
+	struct efa_conn_ep_peer_map_entry *ep_peer_map;
 };
 
+struct efa_conn_ep_peer_map_entry {
+	struct efa_rdm_ep *ep_ptr;
+	struct efa_rdm_peer peer;
+	UT_hash_handle hh;
+};
+
+int efa_conn_ep_peer_map_insert(struct efa_conn *conn,
+				struct efa_conn_ep_peer_map_entry *map_entry);
+
+struct efa_rdm_peer *efa_conn_ep_peer_map_lookup(struct efa_conn *conn,
+						 struct efa_rdm_ep *ep);
+
+void efa_conn_ep_peer_map_remove(struct efa_conn *conn, struct efa_rdm_ep *ep);
+
 int efa_conn_rdm_insert_shm_av(struct efa_av *av, struct efa_conn *conn);
 
 void efa_conn_rdm_deinit(struct efa_av *av, struct efa_conn *conn);

prov/efa/src/rdm/efa_rdm_cq.c

@@ -423,6 +423,8 @@ static void efa_rdm_cq_handle_recv_completion(struct efa_ibv_cq *ibv_cq, struct
 	uint32_t imm_data = 0;
 	bool has_imm_data = false;
 
+	EFA_DBG(FI_LOG_CQ, "Processing receive completion for packet %p\n", pkt_entry);
+
 	if (pkt_entry->alloc_type == EFA_RDM_PKE_FROM_USER_RX_POOL) {
 		assert(ep->user_rx_pkts_posted > 0);
 		ep->user_rx_pkts_posted--;

prov/efa/src/rdm/efa_rdm_ep.h

@@ -118,15 +118,12 @@ struct efa_rdm_ep {
 	/* list of pre-posted recv buffers */
 	struct dlist_entry rx_posted_buf_list;
 
-	/* Hashmap between fi_addr and efa_rdm_peer structs */
-	struct efa_rdm_ep_peer_map_entry *fi_addr_to_peer_map;
-
-	/* Hashmap between implicit peer id and efa_rdm_peer structs */
-	struct efa_rdm_ep_peer_map_entry *fi_addr_to_peer_map_implicit;
-
 	/* bufpool to hold the fi_addr->peer hashmap entries */
 	struct ofi_bufpool *peer_map_entry_pool;
 
+	/**< linked to efa_rdm_ep->ep_peer_list */
+	struct dlist_entry ep_peer_list;
+
 	/* buffer pool for peer reorder circular buffer */
 	struct ofi_bufpool *peer_robuf_pool;
 
@@ -556,27 +553,6 @@ bool efa_rdm_ep_support_unsolicited_write_recv(struct efa_rdm_ep *ep)
 	return ep->extra_info[0] & EFA_RDM_EXTRA_FEATURE_UNSOLICITED_WRITE_RECV;
 }
 
-struct efa_rdm_ep_peer_map_entry {
-	fi_addr_t addr;
-	struct efa_rdm_peer peer;
-	UT_hash_handle hndl;
-};
-
-int
-efa_rdm_ep_peer_map_insert(struct efa_rdm_ep_peer_map_entry **peer_map,
-			   fi_addr_t addr,
-			   struct efa_rdm_ep_peer_map_entry *map_entry);
-struct efa_rdm_peer *
-efa_rdm_ep_peer_map_lookup(struct efa_rdm_ep_peer_map_entry **peer_map,
-			   fi_addr_t addr);
-void efa_rdm_ep_peer_map_remove(struct efa_rdm_ep_peer_map_entry **peer_map,
-				fi_addr_t addr);
-
-void efa_rdm_ep_peer_map_implicit_to_explicit(struct efa_rdm_ep *ep,
-					      struct efa_rdm_peer *peer,
-					      fi_addr_t implicit_fi_addr,
-					      fi_addr_t explicit_fi_addr);
-
 bool efa_rdm_ep_has_unfinished_send(struct efa_rdm_ep *efa_rdm_ep);
 
 #endif