ch4/wait: Optimize wait when requests are local

Optimize progress for a batch of local communication requests by
skipping netmod progress. Shows a 5-10% improvement in ch4/shm
bandwidth measurements on a single node of Cascade Lake.

Author: raffenet

src/mpid/ch4/src/ch4_wait.h

@@ -11,18 +11,28 @@
 MPL_STATIC_INLINE_PREFIX void MPIDI_set_progress_vci(MPIR_Request * req,
                                                      MPID_Progress_state * state)
 {
-    state->flag = MPIDI_PROGRESS_ALL;   /* TODO: check request is_local/anysource */
+    state->flag = MPIDI_PROGRESS_ALL;
 
     int vci = MPIDI_Request_get_vci(req);
 
     state->vci_count = 1;
     state->vci[0] = vci;
+
+#ifndef MPIDI_CH4_DIRECT_NETMOD
+    if (!req->dev.anysrc_partner && MPIDI_REQUEST(req, is_local)) {
+        state->flag &= ~MPIDI_PROGRESS_NM;
+    }
+#endif
 }
 
 MPL_STATIC_INLINE_PREFIX void MPIDI_set_progress_vci_n(int n, MPIR_Request ** reqs,
                                                        MPID_Progress_state * state)
 {
-    state->flag = MPIDI_PROGRESS_ALL;   /* TODO: check request is_local/anysource */
+    state->flag = MPIDI_PROGRESS_ALL;
+    bool nm_progress = false;
+#ifdef MPIDI_CH4_DIRECT_NETMOD
+    nm_progress = true;
+#endif
 
     int idx = 0;
     for (int i = 0; i < n; i++) {
@@ -45,8 +55,16 @@ MPL_STATIC_INLINE_PREFIX void MPIDI_set_progress_vci_n(int n, MPIR_Request ** re
         if (!found) {
             state->vci[idx++] = vci;
         }
+#ifndef MPIDI_CH4_DIRECT_NETMOD
+        nm_progress |= !MPIDI_REQUEST(reqs[i], is_local);
+        nm_progress |= reqs[i]->dev.anysrc_partner != NULL;
+#endif
     }
     state->vci_count = idx;
+
+    if (!nm_progress) {
+        state->flag &= ~MPIDI_PROGRESS_NM;
+    }
 }
 
 /* MPID_Test, MPID_Testall, MPID_Testany, MPID_Testsome */