nonblocking handles with RMA requests

Makefile.am

@@ -18,7 +18,6 @@ libarmci_la_SOURCES = src/buffer.c        \
                       src/internals.c     \
                       src/malloc.c        \
                       src/gmr.c           \
-                      src/gmr-extras.c    \
                       src/message.c       \
                       src/message_gop.c   \
                       src/mutex.c         \

README.md

@@ -159,6 +159,11 @@ Boolean environment variables are enabled when set to a value beginning with
 
   Argument to `usleep()` to pause the progress polling loop.
 
+`ARMCI_USE_REQUEST_ATOMICS` (boolean)
+
+  Switch to request-based RMA (with Rget_accumulate) instead of
+  Fetch_and_op/Compare_and_swap plus a local flush.
+
 ## Noncollective Groups
 
 `ARMCI_NONCOLLECTIVE_GROUPS` (boolean)

src/armci.h

@@ -5,6 +5,9 @@
 #ifndef _ARMCI_H_
 #define _ARMCI_H_
 
+// TODO add to build system
+#define USE_RMA_REQUESTS 1
+
 #include <mpi.h>
 
 #define ARMCI_MPI 3
@@ -64,11 +67,16 @@ int   ARMCI_PutS_flag(void *src_ptr, int src_stride_ar[/*stride_levels*/],
                  int count[/*stride_levels+1*/], int stride_levels,
                  int *flag, int value, int proc);
 
-
 typedef struct armci_hdl_s
 {
+#ifdef USE_RMA_REQUESTS
+    int batch_size;
+    MPI_Request single_request; // used when batch_size=0 (common case)
+    MPI_Request *request_array; // used when batch_size>0
+#else
     int target;    /* we do not actually support individual completion */
     int aggregate;
+#endif
 }
 armci_hdl_t;
 

src/armci_internals.h

@@ -113,6 +113,7 @@ typedef struct {
   int           rma_nocheck;            /* Use MPI_MODE_NOCHECK on synchronization calls that take assertion    */
   int           disable_shm_accumulate; /* Set the disable_shm_accumulate window info key to true               */
   int           use_same_op;            /* Set accumulate_ops=same_op window info key                           */
+  int           use_request_atomics;    /* Use request-based RMA for atomic operations                          */
   char          rma_ordering[20];       /* Set accumulate_ordering=<this> window info key                       */
 
   size_t        memory_limit;           /* upper bound on how much memory ARMCI can allocate                    */
@@ -202,12 +203,12 @@ void ARMCII_Strided_to_dtype(int stride_array[/*stride_levels*/], int count[/*st
                              int stride_levels, MPI_Datatype old_type, MPI_Datatype *new_type);
 
 int ARMCII_Iov_op_dispatch(enum ARMCII_Op_e op, void **src, void **dst, int count, int size,
-    int datatype, int overlapping, int same_alloc, int proc, int blocking);
+    int datatype, int overlapping, int same_alloc, int proc, int blocking, armci_hdl_t * handle);
 
 int ARMCII_Iov_op_batched(enum ARMCII_Op_e op, void **src, void **dst, int count, int elem_count,
-    MPI_Datatype type, int proc, int consrv /* if 1, batched = safe */, int blocking);
+    MPI_Datatype type, int proc, int consrv /* if 1, batched = safe */, int blocking, armci_hdl_t * handle);
 int ARMCII_Iov_op_datatype(enum ARMCII_Op_e op, void **src, void **dst, int count, int elem_count,
-    MPI_Datatype type, int proc, int blocking);
+    MPI_Datatype type, int proc, int blocking, armci_hdl_t * handle);
 
 armcii_iov_iter_t *ARMCII_Strided_to_iov_iter(
                void *src_ptr, int src_stride_ar[/*stride_levels*/],