Adding the circulant graph queued variable ring algorithm for Bcast.

This algorithm achieves better performance than existing algorithms for both small and large message sizes.

The algorithms is based on the circulant graph abstraction and Jesper Larsson Traff's recent paper: https://dl.acm.org/doi/full/10.1145/3735139.
It creates communication schedules around various rings in the circulant graph, then repeats the schedule to pipeline message chunks.
We introduce a FIFO queue for overlapping sends and receives across communication rounds, which particularly benefits small messages.

In the graph below, we show the algorithm's performance for a fixed chunk size (256k) and queue length (24) for various scales on ANL Aurora (N, PPN).
The baseline for this graph is the best-performing algorithm currently in MPICH, so all speedups represent improvements over all algorithms currently in the library.
We note that the performance drops around our selected chunk size (256k).
By tuning the chunk size near this message size, it is possible to achieve a speedup across all message sizes for all scales.

Author: mjwilkins18

src/mpi/coll/bcast/Makefile.mk

@@ -13,6 +13,7 @@ mpi_core_sources +=											\
     src/mpi/coll/bcast/bcast_intra_binomial.c						\
     src/mpi/coll/bcast/bcast_intra_scatter_recursive_doubling_allgather.c	\
     src/mpi/coll/bcast/bcast_intra_scatter_ring_allgather.c		\
+	src/mpi/coll/bcast/bcast_intra_circ_qvring.c        \
     src/mpi/coll/bcast/bcast_intra_smp.c						\
     src/mpi/coll/bcast/bcast_intra_tree.c				\
     src/mpi/coll/bcast/bcast_intra_pipelined_tree.c			\

src/mpi/coll/coll_algorithms.txt

@@ -66,6 +66,9 @@ bcast-intra:
     binomial
     scatter_recursive_doubling_allgather
     scatter_ring_allgather
+    circ_qvring
+        extra_params: chunk_size, q_len
+        cvar_params: CIRC_CHUNK_SIZE, CIRC_Q_LEN
     smp
         restrictions: parent-comm
     tree

src/mpi/coll/cvars.txt

@@ -197,12 +197,33 @@ cvars:
         auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE)
         binomial                                - Force Binomial Tree
         nb                                      - Force nonblocking algorithm
+        circ_qvring                             - Force queued circulant graph algorithm
         smp                                     - Force smp algorithm
         scatter_recursive_doubling_allgather    - Force Scatter Recursive-Doubling Allgather
         scatter_ring_allgather                  - Force Scatter Ring
         pipelined_tree                          - Force tree-based pipelined algorithm
         tree                                    - Force tree-based algorithm
-	
+
+    - name        : MPIR_CVAR_BCAST_CIRC_CHUNK_SIZE
+      category    : COLLECTIVE
+      type        : int
+      default     : 0
+      class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_ALL_EQ
+      description : >-
+        Determines message chunk size for circ_qvring (0 forces single message send)
+
+    - name        : MPIR_CVAR_BCAST_CIRC_Q_LEN
+      category    : COLLECTIVE
+      type        : int
+      default     : 1
+      class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_ALL_EQ
+      description : >-
+        Determines how many bcast rounds can be in flight at once for circ_qvring
+
     - name        : MPIR_CVAR_BCAST_TREE_KVAL
       category    : COLLECTIVE
       type        : int

src/mpi/coll/include/csel_container.h

@@ -58,6 +58,7 @@ typedef enum {
     MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_binomial,
     MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_scatter_recursive_doubling_allgather,
     MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_scatter_ring_allgather,
+    MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_circ_qvring,
     MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_smp,
     MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_tree,
     MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_pipelined_tree,
@@ -325,6 +326,10 @@ typedef struct {
                 int chunk_size;
                 int recv_pre_posted;
             } intra_pipelined_tree;
+            struct {
+                int chunk_size;
+                int q_len;
+            } intra_circ_qvring;
         } bcast;
         struct {
             struct {

src/mpi/coll/src/csel_container.c

@@ -43,6 +43,19 @@ static void parse_container_params(struct json_object *obj, MPII_Csel_container_
             }
             break;
 
+        case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_circ_qvring:
+            {
+                json_object_object_foreach(obj, key, val) {
+                    ckey = MPL_strdup_no_spaces(key);
+                    if (!strncmp(ckey, "chunk_size=", strlen("chunk_size=")))
+                        cnt->u.bcast.intra_circ_qvring.chunk_size = atoi(ckey + strlen("chunk_size="));
+                    else if (!strncmp(ckey, "q_len=", strlen("q_len=")))
+                        cnt->u.bcast.intra_circ_qvring.q_len = atoi(ckey + strlen("q_len="));
+                    MPL_free(ckey);
+                }
+            }
+            break;
+
         case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_tree:
             {
                 json_object_object_foreach(obj, key, val) {
@@ -482,6 +495,8 @@ void *MPII_Create_container(struct json_object *obj)
                 MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_scatter_recursive_doubling_allgather;
         else if (!strcmp(ckey, "algorithm=MPIR_Bcast_intra_scatter_ring_allgather"))
             cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_scatter_ring_allgather;
+        else if (!strcmp(ckey, "algorithm=MPIR_Bcast_intra_circ_qvring"))
+            cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_circ_qvring;
         else if (!strcmp(ckey, "algorithm=MPIR_Bcast_intra_smp"))
             cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_smp;
         else if (!strcmp(ckey, "algorithm=MPIR_Bcast_intra_tree"))

test/mpi/maint/coll_cvars.txt

@@ -355,6 +355,9 @@ algorithms:
             smp
             scatter_recursive_doubling_allgather
             scatter_ring_allgather
+        circ_qvring
+        .MPIR_CVAR_BCAST_CIRC_CHUNK_SIZE=0,1,131072,262144
+        .MPIR_CVAR_BCAST_CIRC_Q_LEN=1,4,8
 	    tree
 		.MPIR_CVAR_BCAST_TREE_TYPE=kary,knomial_1,knomial_2
 		.MPIR_CVAR_BCAST_IS_NON_BLOCKING=1